Fix automated tests for upgrade mode (#2285)

CHANGELOG-1.0.md

@@ -5,3 +5,4 @@
 ### Fixed
 
 - [#2262](https://github.com/epiphany-platform/epiphany/issues/2262) - [Ubuntu] elasticsearch-curator in version 5.8.3 is not available from APT repo
+- [#2259](https://github.com/epiphany-platform/epiphany/issues/2259) - [Upgrade] Automated tests fail when run after upgrade

core/src/epicli/data/common/tests/spec/filebeat/filebeat_spec.rb

@@ -1,24 +1,32 @@
 require 'spec_helper'
 
-elasticsearch_admin_password = readDataYaml("configuration/logging")["specification"]["admin_password"]
-elasticsearch_api_port = 9200
+# Configurable passwords for ES users were introduced in v0.10.0.
+# For testing upgrades, we use default passwords for now but they should be read from filebeat.yml (remote host).
+es_logstash_user_password  = readDataYaml("configuration/logging")["specification"]["logstash_password"] || "logstash"
+es_logstash_user_is_active = readDataYaml("configuration/logging")["specification"]["logstash_user_active"]
+es_logstash_user_is_active = true if es_logstash_user_is_active.nil?
+
+es_kibanaserver_user_password  = readDataYaml("configuration/logging")["specification"]["kibanaserver_password"] || "kibanaserver"
+es_kibanaserver_user_is_active = readDataYaml("configuration/logging")["specification"]["kibanaserver_user_active"]
+es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil?
+
+es_api_port     = 9200
 kibana_api_port = 5601
-kibanaserver_password = readDataYaml("configuration/logging")["specification"]["kibanaserver_password"]
 
-describe 'Checking if Filebeat package is installed' do
+describe 'Check if filebeat package is installed' do
   describe package('filebeat') do
     it { should be_installed }
   end
 end
 
-describe 'Checking if Filebeat service is running' do
+describe 'Check if filebeat service is running' do
   describe service('filebeat') do
     it { should be_enabled }
     it { should be_running }
   end
 end
 
-describe 'Checking Filebeat directories and config files' do
+describe 'Check Filebeat directories and config files' do
   let(:disable_sudo) { false }
   describe file('/etc/filebeat') do
     it { should exist }
@@ -31,7 +39,7 @@
 end
 
 if hostInGroups?("kubernetes_master") || hostInGroups?("kubernetes_node")
-  describe 'Checking extra configuration for master/worker roles - setting Filebeat to be started after Docker' do
+  describe 'Check extra configuration for master/worker roles - setting Filebeat to be started after Docker' do
     describe file("/etc/systemd/system/filebeat.service.d/extra-dependencies.conf") do
       it { should exist }
       it { should be_a_file }
@@ -40,23 +48,28 @@
   end
 end
 
-listInventoryHosts("logging").each do |val|
-  describe 'Checking the connection to the Elasticsearch hosts' do
-    let(:disable_sudo) { false }
-    describe command("curl -k -u admin:#{elasticsearch_admin_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{elasticsearch_api_port}") do
-      it "is expected to be equal" do
-        expect(subject.stdout.to_i).to eq 200
+if es_logstash_user_is_active
+  listInventoryHosts("logging").each do |val|
+    describe 'Check the connection to the Elasticsearch hosts' do
+      let(:disable_sudo) { false }
+      describe command("curl -k -u logstash:#{es_logstash_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do
+        it "is expected to be equal" do
+          expect(subject.stdout.to_i).to eq 200
+        end
       end
     end
   end
 end
 
-listInventoryHosts("kibana").each do |val|
-  describe 'Checking the connection to the Kibana endpoint' do
-    let(:disable_sudo) { false }
-    describe command("curl -u admin:#{kibanaserver_password} -o /dev/null -s -w '%{http_code}' http://#{val}:#{kibana_api_port}/app/kibana") do
-      it "is expected to be equal" do
-        expect(subject.stdout.to_i).to eq 200
+# This test is for optional (manual) command 'filebeat setup --dashboards' (loads Kibana dashboards)
+if es_kibanaserver_user_is_active
+  listInventoryHosts("kibana").each do |val|
+    describe 'Check the connection to the Kibana endpoint' do
+      let(:disable_sudo) { false }
+      describe command("curl -u kibanaserver:#{es_kibanaserver_user_password} -o /dev/null -s -w '%{http_code}' http://#{val}:#{kibana_api_port}/app/kibana") do
+        it "is expected to be equal" do
+          expect(subject.stdout.to_i).to eq 200
+        end
       end
     end
   end

core/src/epicli/data/common/tests/spec/kibana/kibana_spec.rb

@@ -1,24 +1,28 @@
 require 'spec_helper'
 
-elasticsearch_admin_password = readDataYaml("configuration/logging")["specification"]["admin_password"]
-elasticsearch_api_port = 9200
+# Configurable passwords for ES users were introduced in v0.10.0.
+# For testing upgrades, we use the default password for now but it should be read from kibana.yml (remote host).
+es_kibanaserver_user_password  = readDataYaml("configuration/logging")["specification"]["kibanaserver_password"] || "kibanaserver"
+es_kibanaserver_user_is_active = readDataYaml("configuration/logging")["specification"]["kibanaserver_user_active"]
+es_kibanaserver_user_is_active = true if es_kibanaserver_user_is_active.nil?
+
+es_api_port         = 9200
 kibana_default_port = 5601
-kibanaserver_password = readDataYaml("configuration/logging")["specification"]["kibanaserver_password"]
 
-describe 'Checking if Kibana package is installed' do
+describe 'Check if Kibana package is installed' do
   describe package('opendistroforelasticsearch-kibana') do
     it { should be_installed }
   end
 end
 
-describe 'Checking if Kibana service is running' do
+describe 'Check if Kibana service is running' do
   describe service('kibana') do
     it { should be_enabled }
     it { should be_running }
   end
 end
 
-describe 'Checking if Kibana user exists' do
+describe 'Check if Kibana user exists' do
   describe group('kibana') do
     it { should exist }
   end
@@ -28,7 +32,7 @@
   end
 end
 
-describe 'Checking Kibana directories and config files' do
+describe 'Check Kibana directories and config files' do
   describe file('/etc/kibana') do
     it { should exist }
     it { should be_a_directory }
@@ -37,44 +41,45 @@
     it { should exist }
     it { should be_a_file }
   end
-end
-
-describe 'Checking if Kibana log file exists and is not empty' do
-  describe file('/var/log/kibana/kibana.log') do
-    it { should exist }
-    it { should be_a_file }
-    its(:size) { should > 0 }
-  end
   describe file('/etc/logrotate.d/kibana') do
     it { should exist }
     it { should be_a_file }
   end
 end
 
-listInventoryHosts("logging").each do |val|
-  describe 'Checking the connection to the Elasticsearch hosts' do
-    let(:disable_sudo) { false }
-    describe command("curl -k -u admin:#{elasticsearch_admin_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{elasticsearch_api_port}") do
-      it "is expected to be equal" do
-        expect(subject.stdout.to_i).to eq 200
+describe 'Check if non-empty Kibana log file exists' do
+  describe command('find /var/log/kibana -maxdepth 1 -name kibana.log* -size +0 -type f | wc -l') do
+    its(:exit_status) { should eq 0 }
+    its('stdout.to_i') { should > 0 }
+  end
+end
+
+if es_kibanaserver_user_is_active
+  listInventoryHosts("logging").each do |val|
+    describe 'Check the connection to the Elasticsearch hosts' do
+      let(:disable_sudo) { false }
+      describe command("curl -k -u kibanaserver:#{es_kibanaserver_user_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_api_port}") do
+        it "is expected to be equal" do
+          expect(subject.stdout.to_i).to eq 200
+        end
       end
     end
   end
-end
 
-listInventoryHosts("kibana").each do |val|
-  describe 'Checking Kibana app HTTP status code' do
-    let(:disable_sudo) { false }
-    describe command("curl -u admin:#{kibanaserver_password} -o /dev/null -s -w '%{http_code}' http://#{val}:#{kibana_default_port}/app/kibana") do
-      it "is expected to be equal" do
-        expect(subject.stdout.to_i).to eq 200
+  listInventoryHosts("kibana").each do |val|
+    describe 'Check Kibana app HTTP status code' do
+      let(:disable_sudo) { false }
+      describe command("curl -u kibanaserver:#{es_kibanaserver_user_password} -o /dev/null -s -w '%{http_code}' http://#{val}:#{kibana_default_port}/app/kibana") do
+        it "is expected to be equal" do
+          expect(subject.stdout.to_i).to eq 200
+        end
       end
     end
   end
 end
 
 listInventoryHosts("kibana").each do |val|
-  describe 'Checking Kibana health' do
+  describe 'Check Kibana health' do
     let(:disable_sudo) { false }
     describe command("curl http://#{val}:#{kibana_default_port}/api/status") do
       its(:stdout_as_json) { should include('status' => include('overall' => include('state' => 'green'))) }

core/src/epicli/data/common/tests/spec/logging/logging_spec.rb

@@ -1,17 +1,20 @@
 require 'spec_helper'
 
-elasticsearch_admin_password = readDataYaml("configuration/logging")["specification"]["admin_password"]
-elasticsearch_communication_port = 9300
-elasticsearch_rest_api_port = 9200
+# Configurable passwords for ES users were introduced in v0.10.0.
+# For testing upgrades, we use the default password for now but we may switch to TLS auth (after task #2129).
+es_admin_password = readDataYaml("configuration/logging")["specification"]["admin_password"] || "admin"
 
-describe 'Checking if Elasticsearch service is running' do
+es_rest_api_port  = 9200
+es_transport_port = 9300
+
+describe 'Check if Elasticsearch service is running' do
   describe service('elasticsearch') do
     it { should be_enabled }
     it { should be_running }
   end
 end
 
-describe 'Checking if Elasticsearch user exists' do
+describe 'Check if elasticsearch user exists' do
   describe group('elasticsearch') do
     it { should exist }
   end
@@ -21,7 +24,7 @@
   end
 end
 
-describe 'Checking Elasticsearch directories and config files' do
+describe 'Check Elasticsearch directories and config files' do
   let(:disable_sudo) { false }
   describe file('/etc/elasticsearch') do
     it { should exist }
@@ -33,20 +36,20 @@
   end
 end
 
-describe 'Checking if the ports are open' do
+describe 'Check if the ports are open' do
   let(:disable_sudo) { false }
-  describe port(elasticsearch_rest_api_port) do
+  describe port(es_rest_api_port) do
     it { should be_listening }
   end
-  describe port(elasticsearch_communication_port) do
+  describe port(es_transport_port) do
     it { should be_listening }
   end
 end
 
 listInventoryHosts("logging").each do |val|
-  describe 'Checking Elasticsearch nodes status codes' do
+  describe 'Check Elasticsearch nodes status codes' do
     let(:disable_sudo) { false }
-    describe command("curl -k -u admin:#{elasticsearch_admin_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{elasticsearch_rest_api_port}") do
+    describe command("curl -k -u admin:#{es_admin_password} -o /dev/null -s -w '%{http_code}' https://#{val}:#{es_rest_api_port}") do
       it "is expected to be equal" do
         expect(subject.stdout.to_i).to eq 200
       end
@@ -55,9 +58,9 @@
 end
 
 listInventoryHosts("logging").each do |val|
-  describe 'Checking Elasticsearch health' do
+  describe 'Check Elasticsearch health' do
     let(:disable_sudo) { false }
-    describe command("curl -k -u admin:#{elasticsearch_admin_password} https://#{val}:#{elasticsearch_rest_api_port}/_cluster/health?pretty=true") do
+    describe command("curl -k -u admin:#{es_admin_password} https://#{val}:#{es_rest_api_port}/_cluster/health?pretty=true") do
       its(:stdout_as_json) { should include('status' => /green|yellow/) }
       its(:stdout_as_json) { should include('number_of_nodes' => countInventoryHosts("logging")) }
       its(:exit_status) { should eq 0 }