From b5591158ad801f7f958138e4e55da4f5c2a979e6 Mon Sep 17 00:00:00 2001 From: Wajih Yassine Date: Wed, 22 Nov 2023 19:48:25 +0000 Subject: [PATCH 1/5] Add Turbinia e2e GKE test --- turbinia/e2e/e2e-gke.sh | 127 ++++++++++++++++++++++++++++++++ turbinia/e2e/googleclouddisk.sh | 79 -------------------- turbinia/e2e/setup-gce.sh | 45 ----------- turbinia/e2e/teardown-gce.sh | 47 ------------ 4 files changed, 127 insertions(+), 171 deletions(-) create mode 100644 turbinia/e2e/e2e-gke.sh delete mode 100644 turbinia/e2e/googleclouddisk.sh delete mode 100644 turbinia/e2e/setup-gce.sh delete mode 100644 turbinia/e2e/teardown-gce.sh diff --git a/turbinia/e2e/e2e-gke.sh b/turbinia/e2e/e2e-gke.sh new file mode 100644 index 000000000..2a5eee11d --- /dev/null +++ b/turbinia/e2e/e2e-gke.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# Turbinia GKE e2e tests +# The e2e test will test Turbinia googleclouddisk processing +# Requirements: +# - have 'kubectl', 'jq' packages installed +# - have the Turbinia Helm chart deployed and are authenticated to the GKE cluster +# - have the 'turbinia-client' CLI installed + +set -o posix +set -e + +RELEASE="test" +DISK="disk-1" +FAILED=0 +REQUEST_ID=$(uuidgen -rt) +DATE=$(date -I) + +if [ $# -ne 2 ] +then + echo "Not enough arguments supplied, please provide GCP project and zone" + echo "$0 [PROJECT] [ZONE]" + exit 1 +fi + +GCP_PROJECT="$1" +GCP_ZONE="$2" + +echo -n "Started at " +date -Iseconds + +# Back up existing Turbinia config else script will attempt to connect to wrong Turbinia instance +if [ -f ~/.turbinia_api_config.json ] +then + echo "Backing up existing Turbinia config to ~/.turbinia_api_config.json.$DATE" + mv ~/.turbinia_api_config.json ~/.turbinia_api_config.json.$DATE +fi + +# Replace Turbinia config with test config +echo "Writing turbinia config to $HOME/.turbinia_api_config.json..." +cat > $HOME/.turbinia_api_config.json < /dev/null 2>&1 & + +# Run Turbinia googleclouddisk processing +echo "Running Turbinia: turbinia-client submit googleclouddisk --project $GCP_PROJECT --zone $GCP_ZONE --disk_name $DISK --request_id $REQUEST_ID" +turbinia-client submit googleclouddisk --project $GCP_PROJECT --zone $GCP_ZONE --disk_name $DISK --request_id $REQUEST_ID +# To give time for Tasks to populate +sleep 5 + +# Wait until request is complete +req_complete=0 +while [ $req_complete -eq 0 ] +do + req_status=$(turbinia-client status request $REQUEST_ID -j | jq -r '.status') + if [[ $req_status != "running" ]] + then + req_complete=1 + else + echo "Turbinia request $REQUEST_ID is still running. Sleeping for 10 seconds..." + sleep 10 + fi +done + +# Grab all Tasks where successful = false +echo "Checking the status of Turbinia request: $REQUEST_ID" +status=$(turbinia-client status request $REQUEST_ID -j) +task_status=$(echo $status | jq '[.tasks[]] | map({name: .name, id: .id, successful: .successful, worker_name: .worker_name}) | map(select(.successful==false))') +length=$(echo $task_status | jq '. | length') + +# Check if there is a failed Turbinia Task +if [[ $length > 0 ]] +then + echo "A failed Task for Turbinia Request $req has been detected." + echo "Listing failed Tasks..." + # Grab the Task ID + tasks=$(echo $task_status | jq -r '.[] | .id') + FAILED=1 + for t in $tasks + do + echo "Failed Task ID: $t" + turbinia-client status task $t + done + # Grab Turbinia worker logs from the server pod + server=$(kubectl get pods -o name | grep turbinia-server) + workers=$(echo $task_status | jq -r '.[] | .worker_name') + for w in $workers + do + wlogs=$(kubectl exec $server -- find /mnt/turbiniavolume/logs -path "*$w*") + if [ -n $wlogs ] && [ -n $server ] + then + echo "Grabbing logs for Turbinia worker $w" + kubectl exec $server -- cat $wlogs + fi + done +# If no failed Tasks were detected +else + echo "No failed Tasks detected for Turbinia request $req" +fi + +# If there was a failed Task +if [ "$FAILED" != "0" ] +then + echo "Turbinia integration tests failed! Exiting..." + exit 1 +fi + +echo "Turbinia integration tests succeeded!" +echo -n "Ended at " +date -Iseconds + +exit 0 diff --git a/turbinia/e2e/googleclouddisk.sh b/turbinia/e2e/googleclouddisk.sh deleted file mode 100644 index 04f3a6462..000000000 --- a/turbinia/e2e/googleclouddisk.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# This Turbinia e2e test will test the googleclouddisk functionality. -# It will try to process a disk created from the image 'test-disk2'. - - -TURBINIA_CLI="turbiniactl" -LOGS="logs" -MAIN_LOG="$LOGS/main.log" -STATS_LOG="$LOGS/stats.log" -DETAIL_LOG="$LOGS/reqdetails.log" -OUT_TGZ="e2e-test-logs.tgz" -OUT_EXCLUDE="\.ascii|\.tar\.gz|\.plaso|\.csv" -DISK="test-disk2" - -if [ $# -ne 2 ] -then - echo "Not enough arguments supplied, please provide project and zone." - echo "$0 [PROJECT] [ZONE]" - exit 1 -fi - -PROJECT="$1" -ZONE="$2" - -echo -n "Started at " -date -Iseconds - -mkdir $LOGS - -echo "Creating unique request ID...." -REQ_ID=`uuidgen -rt` - -echo "Creating GCE test disk to use in e2e test" -gcloud --project=$PROJECT compute disks create $DISK --image=$DISK --zone=$ZONE - -echo "Executing googlecloudisk e2e test....this takes ~60 minutes!" -$TURBINIA_CLI -d -r $REQ_ID -L $MAIN_LOG -a -w googleclouddisk -d $DISK -z $ZONE - -# When the Turbinia request is finished request the final request statistics. -$TURBINIA_CLI -d status -r $REQ_ID -s > $STATS_LOG 2>&1 - -# Parse out the number of successful and failed tasks. -FAILED=`cat $STATS_LOG | grep Failed | cut -d ":" -f 3 | cut -d ',' -f 1 | tr -d '[:space:]'` -SUCCESS=`cat $STATS_LOG | grep Success | cut -d ":" -f 3 | cut -d ',' -f 1 | tr -d '[:space:]'` - -echo "Results for request ID: $REQ_ID" | tee -a $MAIN_LOG -echo "Failed tasks: $FAILED" | tee -a $MAIN_LOG -echo "Successful tasks: $SUCCESS" | tee -a $MAIN_LOG - -# Output the details, including GCS worker output for the request. -$TURBINIA_CLI -d -a status -r $REQ_ID -R > $DETAIL_LOG 2>&1 - -# Retrieve all test output from GCS and store LOGS folder -echo "Copy all task result files from GCS" -echo "Note: excluding result from StringAsciiTask due to large result file" -cat $DETAIL_LOG|grep "gs://"|tr -d "*\`"|grep -Ev $OUT_EXCLUDE|while read line -do - OUTFILE=`echo "$line"|awk -F/ '{print $(NF-1)"_"$NF}'` - echo "Copying $line to $OUTFILE" - gsutil cp $line $LOGS/$OUTFILE -done - -# Retrieve all worker and server logs from Stackdriver and store in LOGS folder -$TURBINIA_CLI gcplogs -w -o $LOGS -$TURBINIA_CLI gcplogs -s -o $LOGS - -# tgz the log files for debugging purposes -tar -vzcf $OUT_TGZ $LOGS/* - -echo -n "Ended at " -date -Iseconds - -if [ "$FAILED" != "0" ] -then - exit 1 -fi - -exit 0 - diff --git a/turbinia/e2e/setup-gce.sh b/turbinia/e2e/setup-gce.sh deleted file mode 100644 index 49f4ffa11..000000000 --- a/turbinia/e2e/setup-gce.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# This script will -# - configure a GCE Ubuntu 20.04 LTS base image for Turbinia e2e testing. -# - execute Terraform to setup the Turbinia environment. -# Make sure the service account your GCE instance is running under has full API scope -# access and is project owner for Terraform to function correctly. - -echo "Initiate gcloud configuration" - -if [ $# -lt 1 ] -then - echo "Not enough arguments supplied, please provide project." - echo "Any extra arguments will be passed to deploy.sh" - echo "$0 [PROJECT]" - exit 1 -fi - -PROJECT="$1" -shift -EXTRA_ARGS="$@" - -gcloud --project=$PROJECT info - -# Install Terraform -wget -q -O terraform.zip https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_linux_amd64.zip -unzip terraform.zip && cp terraform /usr/local/bin/ && rm terraform.zip terraform - -# Git clone Turbinia Terraform scripts -git clone https://github.com/forseti-security/osdfir-infrastructure.git - -# Deploy Turbinia infrastructure with terraform -# If you see "ERROR: (gcloud.app.create) PERMISSION_DENIED: The caller does not have permission" -# Only GCP Project Owner can create App Engine project -# See documentation: https://cloud.google.com/appengine/docs/standard/python/console/#create -if ! gcloud --project=$PROJECT services list | grep appengine; then - echo "Enable AppEngine API and sleep to make sure service is enabled." - gcloud --project=$PROJECT services enable appengine - sleep 60 -fi - -echo "Setup Terraform Turbinia infrastructure." -export DEVSHELL_PROJECT_ID=$PROJECT -./osdfir-infrastructure/deploy.sh --no-timesketch $EXTRA_ARGS - - diff --git a/turbinia/e2e/teardown-gce.sh b/turbinia/e2e/teardown-gce.sh deleted file mode 100644 index 8ff634984..000000000 --- a/turbinia/e2e/teardown-gce.sh +++ /dev/null @@ -1,47 +0,0 @@ -#/bin/bash -# This script will cleanup and teardown the Turbinia Terraform setup after -# the Turbinia e2e test is finished. -echo "Tear down Terraform Turbinia infrastructure." - - -if [ $# -ne 2 ] -then - echo "Not enough arguments supplied, please provide the project and zone name." - echo "$0 [PROJECT] [ZONE]" - exit 1 -fi - -PROJECT="$1" -ZONE="$2" - -SA_MEMBER="terraform@$PROJECT.iam.gserviceaccount.com" - -echo "Destroying Terraform infrastructure..." -cd ./osdfir-infrastructure/ -terraform destroy --target=module.turbinia -var gcp_project=$PROJECT -auto-approve - -# Remove test and evidence disks (test-disk2 and evidence* disks) -echo "Deleting test and evidence disks..." -gcloud -q --project=$PROJECT compute disks delete test-disk2 --zone=$ZONE -for d in `gcloud compute disks list --uri` -do - DISK=`echo $d | cut -d '/' -f 11` - ZONE=`echo $d | cut -d '/' -f 9` - if [[ $DISK == "evidence"* ]] - then - echo "Deleting $DISK in $ZONE" - gcloud -q --project=$PROJECT compute disks delete $DISK --zone=$ZONE - fi -done - -# Remove IAM bindings and terraform service account -echo "Removing IAM bindings and service account..." -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/editor' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/compute.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/cloudfunctions.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/servicemanagement.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/pubsub.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/storage.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/redis.admin' -gcloud -q --project=$PROJECT projects remove-iam-policy-binding $PROJECT --member=serviceAccount:$SA_MEMBER --role='roles/cloudsql.admin' -gcloud -q --project=$PROJECT iam service-accounts delete $SA_MEMBER From d906f30a8f47ad00ea8aa42469c8f21181c3526d Mon Sep 17 00:00:00 2001 From: Wajih Yassine Date: Wed, 22 Nov 2023 19:52:51 +0000 Subject: [PATCH 2/5] rename e2e script --- turbinia/e2e/{e2e-gke.sh => run-e2e-gke.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename turbinia/e2e/{e2e-gke.sh => run-e2e-gke.sh} (100%) diff --git a/turbinia/e2e/e2e-gke.sh b/turbinia/e2e/run-e2e-gke.sh similarity index 100% rename from turbinia/e2e/e2e-gke.sh rename to turbinia/e2e/run-e2e-gke.sh From cc9ae52242792f68566fdb99209ddaa02c4aad66 Mon Sep 17 00:00:00 2001 From: Wajih Yassine Date: Wed, 22 Nov 2023 20:45:47 +0000 Subject: [PATCH 3/5] list turbinia config sleep on port-forward --- turbinia/e2e/run-e2e-gke.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/turbinia/e2e/run-e2e-gke.sh b/turbinia/e2e/run-e2e-gke.sh index 2a5eee11d..2a8403cbf 100644 --- a/turbinia/e2e/run-e2e-gke.sh +++ b/turbinia/e2e/run-e2e-gke.sh @@ -56,11 +56,17 @@ echo "Starting e2e test for Turbinia..." # Forward k8s services echo "Forwarding Turbinia API k8s $RELEASE service" kubectl --namespace default port-forward service/$RELEASE-turbinia 8000:8000 > /dev/null 2>&1 & +# Give time before submitting request to service +sleep 5 + +# List Turbinia config +echo "Listing Turbinia config..." +turbinia-cient config list # Run Turbinia googleclouddisk processing echo "Running Turbinia: turbinia-client submit googleclouddisk --project $GCP_PROJECT --zone $GCP_ZONE --disk_name $DISK --request_id $REQUEST_ID" turbinia-client submit googleclouddisk --project $GCP_PROJECT --zone $GCP_ZONE --disk_name $DISK --request_id $REQUEST_ID -# To give time for Tasks to populate +# Give time for Tasks to populate sleep 5 # Wait until request is complete From 0a3b41b884cb34bea1a5839481ce4a27c8a81ad4 Mon Sep 17 00:00:00 2001 From: Wajih Yassine Date: Wed, 22 Nov 2023 20:52:59 +0000 Subject: [PATCH 4/5] fix spelling --- turbinia/e2e/run-e2e-gke.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbinia/e2e/run-e2e-gke.sh b/turbinia/e2e/run-e2e-gke.sh index 2a8403cbf..cc8d140ad 100644 --- a/turbinia/e2e/run-e2e-gke.sh +++ b/turbinia/e2e/run-e2e-gke.sh @@ -61,7 +61,7 @@ sleep 5 # List Turbinia config echo "Listing Turbinia config..." -turbinia-cient config list +turbinia-client config list # Run Turbinia googleclouddisk processing echo "Running Turbinia: turbinia-client submit googleclouddisk --project $GCP_PROJECT --zone $GCP_ZONE --disk_name $DISK --request_id $REQUEST_ID" From 199a7744c5911d5b9b3ecae0934f58fd3fb1d389 Mon Sep 17 00:00:00 2001 From: Wajih Yassine Date: Thu, 30 Nov 2023 00:29:16 +0000 Subject: [PATCH 5/5] Address review comments --- turbinia/e2e/run-e2e-gke.sh | 49 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/turbinia/e2e/run-e2e-gke.sh b/turbinia/e2e/run-e2e-gke.sh index cc8d140ad..692590833 100644 --- a/turbinia/e2e/run-e2e-gke.sh +++ b/turbinia/e2e/run-e2e-gke.sh @@ -2,9 +2,12 @@ # Turbinia GKE e2e tests # The e2e test will test Turbinia googleclouddisk processing # Requirements: -# - have 'kubectl', 'jq' packages installed -# - have the Turbinia Helm chart deployed and are authenticated to the GKE cluster +# TODO: Have seperate checks for each of the requirements +# - have 'kubectl', 'jq', and 'uuid-runtime' packages installed # - have the 'turbinia-client' CLI installed +# - have authenticated to your GKE cluster via `gcloud container clusters get-credentials [clustername] --zone [zone] --project [project_name]` +# - have the Turbinia Helm chart deployed and the Helm release name matching the $RELEASE variable +# - have a GCP disk created that matches the $DISK variable name set -o posix set -e @@ -36,8 +39,8 @@ then fi # Replace Turbinia config with test config -echo "Writing turbinia config to $HOME/.turbinia_api_config.json..." -cat > $HOME/.turbinia_api_config.json < ~/.turbinia_api_config.json < $HOME/.turbinia_api_config.json < 0 ]] then - echo "A failed Task for Turbinia Request $req has been detected." - echo "Listing failed Tasks..." + echo "A failed Task for Turbinia Request $req has been detected." + echo "Listing failed Tasks..." # Grab the Task ID tasks=$(echo $task_status | jq -r '.[] | .id') - FAILED=1 - for t in $tasks + FAILED=1 + for t in $tasks do - echo "Failed Task ID: $t" - turbinia-client status task $t - done + echo "Failed Task ID: $t" + turbinia-client status task $t + done # Grab Turbinia worker logs from the server pod server=$(kubectl get pods -o name | grep turbinia-server) workers=$(echo $task_status | jq -r '.[] | .worker_name') @@ -119,6 +119,13 @@ else echo "No failed Tasks detected for Turbinia request $req" fi +# Restore previous Turbinia config +if [ -f ~/.turbinia_api_config.json.$DATE ] +then + echo "Restoring previous Turbinia config from ~/.turbinia_api_config.json.$DATE" + mv ~/.turbinia_api_config.json.$DATE ~/.turbinia_api_config.json +fi + # If there was a failed Task if [ "$FAILED" != "0" ] then