Skip to content

Commit e613504

Browse files
chaitanyaphalakDonald Tregonning
chaitanyaphalak
authored and
Donald Tregonning
committed
Release/1.2.0 (#169)
updates and fixes for release 1.2.0
1 parent ec00d8f commit e613504

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+615
-178
lines changed

‎.circleci/build_sck_artifacts_for_release.sh‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,38 +2,45 @@
22
set -e
33
TAG=`cat VERSION`
44

5-
function replace_generic_version ()
6-
{
7-
file="$1"
8-
_line=`awk '/version:/{print NR;exit}' $file`
9-
replacement="version: $TAG"
10-
# Escape backslash, forward slash and ampersand for use as a sed replacement.
11-
replacement_escaped=$( echo "$replacement" | sed -e 's/[\/&]/\\&/g' )
12-
sed -i "${_line}s/.*/$replacement_escaped/" "$file"
13-
}
5+
# Install yq yaml parser
6+
wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64
7+
sudo chmod +x yq_linux_amd64
8+
sudo mv yq_linux_amd64 /usr/local/bin/yq
149

15-
repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging"
16-
"helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" )
17-
sub_repos_array=( "helm-chart/splunk-kubernetes-logging"
18-
"helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" )
10+
yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name splunk/fluentd-hec:1.1.1
11+
yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name splunk/k8s-metrics:1.1.1
12+
yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name splunk/k8s-metrics-aggr:1.1.0
13+
yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name splunk/kube-objects:1.1.0
1914

20-
for repo in "${repos_array[@]}"
21-
do
22-
filename="${repo}/Chart.yaml"
23-
replace_generic_version $filename
24-
done
15+
yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION
16+
yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION
17+
yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION
18+
yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION
19+
20+
yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml appVersion $VERSION
21+
yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml appVersion $VERSION
22+
yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml appVersion $VERSION
23+
yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml appVersion $VERSION
24+
25+
yq w -i helm-chart/splunk-kubernetes-logging/requirements.yaml dependencies.splunk-kubernetes-logging.version $VERSION
26+
yq w -i helm-chart/splunk-kubernetes-metrics/requirements.yaml dependencies.splunk-kubernetes-metrics.version $VERSION
27+
yq w -i helm-chart/splunk-kubernetes-objects/requirements.yaml dependencies.splunk-kubernetes-objects.version $VERSION
2528

2629
mkdir helm-artifacts-release
2730
if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then
2831
rm -rf helm-chart/splunk-connect-for-kubernetes/charts
2932
fi
3033
mkdir helm-chart/splunk-connect-for-kubernetes/charts
3134

35+
sub_repos_array=( "helm-chart/splunk-kubernetes-logging"
36+
"helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" )
3237
for sub_repo in "${sub_repos_array[@]}"
3338
do
3439
cp -rp $sub_repo helm-chart/splunk-connect-for-kubernetes/charts
3540
done
3641

42+
repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging"
43+
"helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" )
3744
for repo in "${repos_array[@]}"
3845
do
3946
helm package -d helm-artifacts-release $repo

‎.circleci/config.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1
1616
sudo apt-get install python3-pip python3-dev > /dev/null 2>&1
1717
sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1
18-
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
18+
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3
1919
helm init -c
2020
echo "Installing aws cli..."
2121
sudo pip install awscli > /dev/null 2>&1
@@ -59,7 +59,7 @@ jobs:
5959
sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1
6060
sudo apt-get install python3-pip python3-dev > /dev/null 2>&1
6161
sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1
62-
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
62+
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3
6363
helm init -c
6464
echo "Installing aws cli..."
6565
sudo pip install awscli > /dev/null 2>&1

‎.circleci/performance/perf_deploy_sck.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ function setup_kubeclient ()
2727
function deploy_sck ()
2828
{
2929
print_msg "Deploying SCK"
30-
setup_kubeclient
30+
setup_kubeclient || true
3131
print_msg "Installing the SCK build artifacts on the kubernetes cluster"
3232
helm install --name=perf-test -f .circleci/performance/perf_test_sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz
3333
}

‎.circleci/performance/run_perf_scenarios.py‎

Lines changed: 212 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,150 @@
1313

1414
# Lookup configurables for datagen - https://github.com/dtregonning/kafka-data-gen
1515
DATAGEN_PERF_CASES = [
16-
{
17-
'deployment_name': 'perf-test-datagen',
18-
'namespace': 'default',
19-
'number_of_datagen': 3,
20-
'message_count': 10000,
21-
'message_size': 256,
22-
'eps': 500,
23-
'sleep_duration': 120
24-
},
16+
#{
17+
# 'deployment_name': 'perf-test-datagen',
18+
# 'namespace': 'default',
19+
# 'number_of_datagen': 10,
20+
# 'message_count': 100000,
21+
# 'message_size': 128,
22+
# 'eps': 100,
23+
# 'sleep_duration': 1800
24+
#},
25+
#{
26+
# 'deployment_name': 'perf-test-datagen',
27+
# 'namespace': 'default',
28+
# 'number_of_datagen': 10,
29+
# 'message_count': 100000,
30+
# 'message_size': 256,
31+
# 'eps': 100,
32+
# 'sleep_duration': 1800
33+
#},
34+
#{
35+
# 'deployment_name': 'perf-test-datagen',
36+
# 'namespace': 'default',
37+
# 'number_of_datagen': 10,
38+
# 'message_count': 100000,
39+
# 'message_size': 512,
40+
# 'eps': 100,
41+
# 'sleep_duration': 1800
42+
#},
43+
#{
44+
# 'deployment_name': 'perf-test-datagen',
45+
# 'namespace': 'default',
46+
# 'number_of_datagen': 10,
47+
# 'message_count': 100000,
48+
# 'message_size': 1024,
49+
# 'eps': 100,
50+
# 'sleep_duration': 1800
51+
# },
52+
#{
53+
# 'deployment_name': 'perf-test-datagen',
54+
# 'namespace': 'default',
55+
# 'number_of_datagen': 50,
56+
# 'message_count': 100000,
57+
# 'message_size': 128,
58+
# 'eps': 100,
59+
# 'sleep_duration': 1800
60+
# },
61+
# {
62+
# 'deployment_name': 'perf-test-datagen',
63+
# 'namespace': 'default',
64+
# 'number_of_datagen': 50,
65+
# 'message_count': 100000,
66+
# 'message_size': 256,
67+
# 'eps': 100,
68+
# 'sleep_duration': 1800
69+
# },
70+
# {
71+
# 'deployment_name': 'perf-test-datagen',
72+
# 'namespace': 'default',
73+
# 'number_of_datagen': 50,
74+
# 'message_count': 100000,
75+
# 'message_size': 512,
76+
# 'eps': 100,
77+
# 'sleep_duration': 1800
78+
# },
79+
# {
80+
# 'deployment_name': 'perf-test-datagen',
81+
# 'namespace': 'default',
82+
# 'number_of_datagen': 50,
83+
# 'message_count': 100000,
84+
# 'message_size': 1024,
85+
# 'eps': 100,
86+
# 'sleep_duration': 1800
87+
# }
88+
# {
89+
# 'deployment_name': 'perf-test-datagen',
90+
# 'namespace': 'default',
91+
# 'number_of_datagen': 100,
92+
# 'message_count': 100000,
93+
# 'message_size': 128,
94+
# 'eps': 100,
95+
# 'sleep_duration': 3600
96+
# },
97+
# {
98+
# 'deployment_name': 'perf-test-datagen',
99+
# 'namespace': 'default',
100+
# 'number_of_datagen': 100,
101+
# 'message_count': 100000,
102+
# 'message_size': 256,
103+
# 'eps': 100,
104+
# 'sleep_duration': 3600
105+
# },
106+
# {
107+
# 'deployment_name': 'perf-test-datagen',
108+
# 'namespace': 'default',
109+
# 'number_of_datagen': 100,
110+
# 'message_count': 100000,
111+
# 'message_size': 512,
112+
# 'eps': 100,
113+
# 'sleep_duration': 3600
114+
# },
115+
# {
116+
# 'deployment_name': 'perf-test-datagen',
117+
# 'namespace': 'default',
118+
# 'number_of_datagen': 100,
119+
# 'message_count': 100000,
120+
# 'message_size': 1024,
121+
# 'eps': 100,
122+
# 'sleep_duration': 3600
123+
#
124+
# {
125+
# 'deployment_name': 'perf-test-datagen',
126+
# 'namespace': 'default',
127+
# 'number_of_datagen': 100,
128+
# 'message_count': 225000 ,
129+
# 'message_size': 1024,
130+
# 'eps': 250,
131+
# 'sleep_duration': 3600
132+
# },
133+
# {
134+
# 'deployment_name': 'perf-test-datagen',
135+
# 'namespace': 'default',
136+
# 'number_of_datagen': 100,
137+
# 'message_count': 225000 ,
138+
# 'message_size': 1024,
139+
# 'eps': 250,
140+
# 'sleep_duration': 3600
141+
# },
142+
# {
143+
# 'deployment_name': 'perf-test-datagen',
144+
# 'namespace': 'default',
145+
# 'number_of_datagen': 100,
146+
# 'message_count': 225000 ,
147+
# 'message_size': 1024,
148+
# 'eps': 250,
149+
# 'sleep_duration': 3600
150+
# },
151+
{
152+
'deployment_name': 'perf-test-datagen',
153+
'namespace': 'default',
154+
'number_of_datagen': 10,
155+
'message_count': 1000 ,
156+
'message_size': 1024,
157+
'eps': 100,
158+
'sleep_duration': 3600
159+
}
25160
]
26161

27162
# Lookup configurables for fluent and kubernetes
@@ -30,18 +165,74 @@
30165
CONNECTOR_PERF_CASES = [
31166
{
32167
'buffer_type': 'memory',
33-
'total_limit_size': '400m',
34-
'chunk_limit_size': '500m',
35-
'chunk_limit_records': 1,
36-
'flush_interval': '1s',
168+
'total_limit_size': '600m',
169+
'chunk_limit_size': '200m',
170+
'chunk_limit_records': 512000,
171+
'flush_interval': '10s',
37172
'flush_thread_count': 1,
38173
'overflow_action': 'block',
39174
'retry_max_times': 3,
40-
'limits_cpu': '1000m',
41-
'limits_memory': '1000Mi',
42-
'requests_cpu': '500m',
43-
'requests_memory': '500Mi',
175+
'limits_cpu': '2000m',
176+
'limits_memory': '2000Mi',
177+
'requests_cpu': '1000m',
178+
'requests_memory': '1000Mi',
44179
},
180+
# {
181+
# 'buffer_type': 'memory',
182+
# 'total_limit_size': '600m',
183+
# 'chunk_limit_size': '200m',
184+
# 'chunk_limit_records': 10000,
185+
# 'flush_interval': '5s',
186+
# 'flush_thread_count': 1,
187+
# 'overflow_action': 'block',
188+
# 'retry_max_times': 3,
189+
# 'limits_cpu': '2000m',
190+
# 'limits_memory': '2000Mi',
191+
# 'requests_cpu': '1000m',
192+
# 'requests_memory': '1000Mi',
193+
# },
194+
# {
195+
# 'buffer_type': 'memory',
196+
# 'total_limit_size': '400m',
197+
# 'chunk_limit_size': '100m',
198+
# 'chunk_limit_records': 10000,
199+
# 'flush_interval': '5s',
200+
# 'flush_thread_count': 1,
201+
# 'overflow_action': 'block',
202+
# 'retry_max_times': 3,
203+
# 'limits_cpu': '2000m',
204+
# 'limits_memory': '2000Mi',
205+
# 'requests_cpu': '500m',
206+
# 'requests_memory': '500Mi',
207+
# },
208+
# {
209+
# 'buffer_type': 'memory',
210+
# 'total_limit_size': '400m',
211+
# 'chunk_limit_size': '100m',
212+
# 'chunk_limit_records': 10000,
213+
# 'flush_interval': '5s',
214+
# 'flush_thread_count': 1,
215+
# 'overflow_action': 'block',
216+
# 'retry_max_times': 3,
217+
# 'limits_cpu': '2000m',
218+
# 'limits_memory': '2000Mi',
219+
# 'requests_cpu': '1000m',
220+
# 'requests_memory': '1000Mi',
221+
# },
222+
# {
223+
# 'buffer_type': 'file',
224+
# 'total_limit_size': '600m',
225+
# 'chunk_limit_size': '200m',
226+
# 'chunk_limit_records': 10000,
227+
# 'flush_interval': '5s',
228+
# 'flush_thread_count': 1,
229+
# 'overflow_action': 'block',
230+
# 'retry_max_times': 3,
231+
# 'limits_cpu': '2000m',
232+
# 'limits_memory': '2000Mi',
233+
# 'requests_cpu': '500m',
234+
# 'requests_memory': '500Mi',
235+
# },
45236
]
46237

47238

@@ -131,7 +322,10 @@ def teardown_connector_and_datagen(test_case_datagen, test_case_connector):
131322

132323
def wait_for_connector_do_data_collection_injection(test_case_datagen):
133324
logging.info('Sleeping for data collection')
134-
time.sleep(int(test_case_datagen['sleep_duration']))
325+
for second in range(int(test_case_datagen['sleep_duration']), 0, -1):
326+
if(second % 60 == 0):
327+
logging.info('Sleeping ' + str(second) + '........')
328+
time.sleep(1)
135329

136330

137331
def _do_perf():

‎.circleci/update_sck_config_for_release.sh‎

Lines changed: 5 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -8,44 +8,19 @@ wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64
88
sudo chmod +x yq_linux_amd64
99
sudo mv yq_linux_amd64 /usr/local/bin/yq
1010

11-
## Modify docker images to be used
12-
#image_name=$(echo "splunk/fluentd-hec:$VERSION")
13-
#yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name $image_name
14-
#
15-
#image_name=$(echo "splunk/k8s-metrics:$VERSION")
16-
#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name $image_name
17-
#
18-
#image_name=$(echo "splunk/k8s-metrics-aggr:$VERSION")
19-
#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name $image_name
20-
#
21-
#image_name=$(echo "splunk/kube-objects:$VERSION")
22-
#yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name $image_name
23-
#
24-
#yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION
25-
#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION
26-
#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION
27-
#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION
28-
#
29-
#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml dependencies.splunk-kubernetes-logging.version $VERSION
30-
#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml dependencies.splunk-kubernetes-metrics.version $VERSION
31-
#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml dependencies.splunk-kubernetes-objects.version $VERSION
32-
33-
11+
# setup necessary for functional tests
3412
# Modify splunk environment values
3513
yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST
3614
yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN
3715

3816
# Pull docker images locally
39-
docker pull splunk/fluentd-hec:1.1.0
40-
docker pull splunk/k8s-metrics:1.1.0
17+
docker pull splunk/fluentd-hec:1.1.1
18+
docker pull splunk/k8s-metrics:1.1.1
4119
docker pull splunk/k8s-metrics-aggr:1.1.0
4220
docker pull splunk/kube-objects:1.1.0
4321

4422
# Modify docker images to be used
45-
yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.0
46-
47-
yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.0
48-
23+
yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.1
24+
yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.1
4925
yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr:1.1.0
50-
5126
yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects:1.1.0

0 commit comments

Comments
 (0)