System test fixes

README.md

@@ -52,6 +52,12 @@ pytest --pdb -s cosmo_tester/test_suites/general/simple_deployment_test.py::test
 --pdb is recommended for manual test runs as this will pause test execution on failure and may allow you to gain valuable insight into the cause of the failure.
 -s is recommended in order to ensure all test output is shown.
 
+#### Tests which use an external DB
+Some of our tests (currently only `cosmo_tester/test_suites/cluster/external_component_cluster_test.py::test_upgrade_external_db`) use an external database.
+The FQDN and password of the database should be provided as environment variables `EXTDB_FQDN=...` and `EXTDB_PSWD=...` when running the test manually.
+Developers may refer to the `pipelines-k8s/system-tests` pipeline in our `cloudify-build-system` repo for the values of our test-db on RDS.
+
+
 ## Using the config in tests
 There are two supported ways of accessing the config within tests.
 

aws_cleanup

@@ -12,6 +12,7 @@ except ImportError:
     print('You need to pip install boto3==1.12.13')
     sys.exit(1)
 
+TEST_GENERATED = '*pytest*'
 NOT_TERMINATED = [
     'pending', 'running', 'shutting-down', 'stopping', 'stopped'
 ]
@@ -164,6 +165,16 @@ def get_vpcs(client, owner):
         ],
     )['Vpcs']
 
+def get_all_vpcs(client):
+    vpcs = []
+    for vpc in client.describe_vpcs(
+        Filters = [
+            {'Name' : 'tag:CreatedBy' , 'Values' : [TEST_GENERATED]}
+        ]
+    )['Vpcs']:
+        if not 'Name' in [ k['Key'] for k in vpc['Tags'] ]:
+            vpcs.append(vpc)
+    return vpcs
 
 def get_subnets(client, vpc):
     return client.describe_subnets(
@@ -422,8 +433,32 @@ def delete_resources(client, resources_by_test):
             # investigate by hand if the script can't handle it.
             client.delete_vpc(VpcId=vpc)
 
+def collect_resources_from_unknown_test(client, owner):
+    vpcs = get_all_vpcs(client)
+    for i, _ in enumerate(vpcs):
+        vpcs[i]['Tags'].append( { 'Key' : 'Name', 'Value' : 'unknowntest'+str(i) + "_" \
+                                 +  str(datetime.now().strftime('%Y%m%d%H%M%S'))} )
+    resources_by_test = {}
+    for i, vpc in enumerate(vpcs):
+        vpc_id = vpc['VpcId']
 
-def main(older_than, newer_than, owner, delete):
+        instances = get_instances(client, vpc_id)
+        interfaces = get_interfaces(client, vpc_id)
+        subnets = get_subnets(client, vpc_id)
+        route_tables = get_route_tables(client, vpc_id)
+        security_groups = get_security_groups(client, vpc_id)
+        internet_gateways = get_internet_gateways(client, vpc_id)
+
+        group_by_previous_tests(instances, interfaces,
+                                subnets, route_tables,
+                                internet_gateways,
+                                security_groups,
+                                vpc, resources_by_test)
+
+    output_resources_by_test(resources_by_test)
+    return resources_by_test
+
+def main(older_than, newer_than, owner, delete, unknown):
     conf = load_config()
     client = get_ec2_client(conf)
 
@@ -433,6 +468,7 @@ def main(older_than, newer_than, owner, delete):
     resources_by_test = {}
 
     vpcs = get_vpcs(client, owner)
+
     for vpc in vpcs:
         vpc_id = vpc['VpcId']
         instances = get_instances(client, vpc_id)
@@ -441,7 +477,7 @@ def main(older_than, newer_than, owner, delete):
         route_tables = get_route_tables(client, vpc_id)
         security_groups = get_security_groups(client, vpc_id)
         internet_gateways = get_internet_gateways(client, vpc_id)
-
+        
         group_by_previous_tests(instances, interfaces,
                                 subnets, route_tables,
                                 internet_gateways,
@@ -453,13 +489,15 @@ def main(older_than, newer_than, owner, delete):
                                              resources_by_test)
 
     filter_test_groups(resources_by_test, older_than, newer_than)
-
     output_resources_by_test(resources_by_test)
-
+    
     if delete:
         print('Deleting')
         delete_resources(client, resources_by_test)
-
+        if unknown: 
+            print('Deleting resources from unkown tests')
+            unknown_resources = collect_resources_from_unknown_test(client, owner)
+            delete_resources(client, unknown_resources)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -492,8 +530,13 @@ if __name__ == '__main__':
         action='store_true',
         default=False,
     )
+    parser.add_argument(
+        '-u', '--unknown',
+        help='Delete discovered resources from unkown tests.',
+        action='store_true',
+        default=False,
+    )
 
     args = parser.parse_args()
 
-    main(older_than=args.older, newer_than=args.newer, owner=args.owner,
-         delete=args.delete)
+    main(older_than=args.older, newer_than=args.newer, owner=args.owner, delete=args.delete, unknown=args.unknown)

cosmo_tester/config_schemas/base.yaml

@@ -7,4 +7,4 @@ target_platform:
   valid_values: [openstack, aws]
 testing_version:
   description: Which manager version we're testing. Note that this is expected to be in the form <version number>-<identifier>. Bad things may happen without the hyphen.
-  default: 7.0.0-.dev1
+  default: 7.1.0-.dev1

cosmo_tester/config_schemas/cfy_cluster_manager.yaml

@@ -1,4 +1,5 @@
 namespace: cfy_cluster_manager
 rpm_path:
   description: The cfy_cluster_manager RPM path to install.
-  default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.4/ga-release/cloudify-cluster-manager-1.1.4-ga.el7.x86_64.rpm'
+  default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.6/ga-release/cloudify-cluster-manager-1.1.6-ga.el7.x86_64.rpm'
+

cosmo_tester/config_schemas/platform_aws.yaml

@@ -28,7 +28,7 @@ ubuntu_14_04_image:
   default: ami-005af4c3162f495fa
 ubuntu_16_04_image:
   description: Image to use for Ubuntu 16.04 on this platform.
-  default: ami-0a9aac550bc5711d3
+  default: ami-0f29c8402f8cce65c
 ubuntu_18_04_image:
   description: Image to use for Ubuntu 18.04 on this platform.
   default: ami-02f0341ac93c96375

cosmo_tester/framework/examples.py

@@ -144,9 +144,10 @@ def install(self):
         self.execute('install')
         self.installed = True
 
-    def uninstall(self, check_files_are_deleted=True, delete_dep=True):
+    def uninstall(self, check_files_are_deleted=True, delete_dep=True,
+                  parameters=None):
         self.logger.info('Cleaning up example.')
-        self.execute('uninstall')
+        self.execute('uninstall', parameters=parameters)
         self.installed = False
         if check_files_are_deleted:
             self.check_all_test_files_deleted()

cosmo_tester/framework/test_hosts.py

@@ -854,6 +854,10 @@ def download_rest_ca(self, force=False):
             '/etc/cloudify/ssl/cloudify_internal_ca_cert.pem',
             self.api_ca_path,
         )
+        # close the current restclient session to force making a new connection
+        # using the new certificate, on first use after this call
+        if self.client:
+            self.client._client._session.close()
 
     @only_manager
     def clean_local_rest_ca(self):
@@ -922,7 +926,8 @@ def _is_manager_image_type(self):
     def _is_rhel8_supported(self):
         if self.image_type == 'master':
             return True
-        if parse_version(self.image_type) >= parse_version('6.4.0'):
+        if parse_version(self.image_type.split('-')[0]) \
+                >= parse_version('6.4.0'):
             return True
         return False
 
@@ -986,6 +991,7 @@ def rsync_backup(self):
             'Creating Rsync backup for host {}. Might take up to 5 '
             'minutes...'.format(self.deployment_id))
         self.run_command("mkdir /cfy_backup", use_sudo=True)
+        self.run_command("chmod o+r /cfy_backup", use_sudo=True)
         rsync_backup_file = self._tmpdir / 'rsync_backup_{0}'.format(
             self.ip_address)
         locations = ' '.join(RSYNC_LOCATIONS)
@@ -1134,7 +1140,7 @@ def __init__(self,
         else:
             self.server_flavor = self._test_config.platform['linux_size']
 
-    def create(self):
+    def create(self, use_fqdn=False):
         """Creates the infrastructure for a Cloudify manager."""
         self._logger.info('Creating image based cloudify instances: '
                           '[number_of_instances=%d]', len(self.instances))
@@ -1173,7 +1179,7 @@ def create(self):
                                            test_identifier,
                                            instance.is_manager,
                                            instance.image_type)
-            self._finish_deploy_test_vms()
+            self._finish_deploy_test_vms(use_fqdn=use_fqdn)
 
             for instance in self.instances:
                 if instance.is_manager and not instance.bootstrappable:
@@ -1565,7 +1571,7 @@ def _populate_aws_platform_properties(self):
 
         self._platform_resource_ids = resource_ids
 
-    def _finish_deploy_test_vms(self):
+    def _finish_deploy_test_vms(self, use_fqdn=False):
         node_instances = {}
         for vm_id, details in self._test_vm_installs.items():
             execution, index = details
@@ -1580,6 +1586,7 @@ def _finish_deploy_test_vms(self):
             self._update_instance(
                 index,
                 node_instance,
+                use_fqdn=use_fqdn,
             )
 
             node_instances[index] = node_instance
@@ -1607,11 +1614,16 @@ def _finish_undeploy_test_vms(self):
             util.delete_deployment(self._infra_client, vm_id,
                                    self._logger)
 
-    def _update_instance(self, server_index, node_instance):
+    def _update_instance(self, server_index, node_instance, use_fqdn=False):
         instance = self.instances[server_index]
         runtime_props = node_instance['runtime_properties']
 
         public_ip_address = runtime_props['public_ip_address']
+        if use_fqdn and self._test_config['target_platform'] == 'aws':
+            zone = runtime_props['resource']['Placement']['AvailabilityZone']
+            public_ip_address = f"ec2-{public_ip_address.replace('.', '-')}."\
+                                f"{zone[:-1]}.compute.amazonaws.com"
+
         private_ip_address = runtime_props['ipv6_address'] if self.ipv6_net \
             else runtime_props['ip']
 

cosmo_tester/framework/util.py

@@ -598,6 +598,8 @@ def delete_deployment(client, deployment_id, logger):
     for _ in range(40):
         found = False
         deployments = client.deployments.list()
+        if not deployments:
+            break
         for deployment in deployments:
             if deployment['id'] == deployment_id:
                 found = True

cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml

@@ -1,7 +1,7 @@
 tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
-  - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml
+  - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml
   - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml
 
 inputs:

cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml

@@ -1,7 +1,7 @@
 tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
-  - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml
+  - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml
   - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml
 
 inputs:

cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml

@@ -2,7 +2,7 @@
 tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
-  - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml
+  - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml
   - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml
 
 inputs:

cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml

@@ -2,7 +2,7 @@
 tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
-  - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml
+  - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml
   - plugin:cloudify-openstack-plugin
 
 inputs:

cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml

@@ -2,7 +2,7 @@
 tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
-  - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml
+  - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml
   - plugin:cloudify-openstack-plugin
 
 inputs:

cosmo_tester/test_suites/agent/__init__.py

@@ -1,3 +1,5 @@
+from cloudify.models_states import AgentState
+
 from cosmo_tester.framework.test_hosts import Hosts, VM
 
 
@@ -25,8 +27,13 @@ def get_test_prerequisites(ssh_key, module_tmpdir, test_config, logger,
 def validate_agent(manager, example, test_config,
                    broken_system=False, install_method='remote',
                    upgrade=False):
-    agents = list(manager.client.agents.list(tenant_name=example.tenant,
-                                             _all_tenants=True))
+    agents = list(
+        manager.client.agents.list(
+            tenant_name=example.tenant,
+            state=AgentState.STARTED,
+            _all_tenants=True,
+        )
+    )
     instances = list(
         manager.client.node_instances.list(
             tenant_name=example.tenant, node_id='vm',

cosmo_tester/test_suites/auth/test_openldap.py

@@ -243,6 +243,13 @@ def _configure_openldap(host, logger):
 replace: olcTLSCACertificateFile
 olcTLSCACertificateFile: /etc/ssl/certs/ldapcacert.crt''',
     )
+    host.run_command(
+        'chgrp ldap'
+        ' /etc/openldap/certs/ldap-key.pem'
+        ' /etc/openldap/certs/ldap-cert.pem'
+        ' /etc/ssl/certs/ldapcacert.crt',
+        use_sudo=True,
+    )
     host.run_command(
         'ldapmodify -Y EXTERNAL -H ldapi:/// -f /tmp/certs.ldif',
         use_sudo=True,

cosmo_tester/test_suites/auth/test_saml.py

@@ -61,7 +61,7 @@ def test_saml_auth(image_based_manager, logger):
 
 def _activate_saml_auth(manager, logger):
     logger.info('Putting SAML cert in location and restarting rest service')
-    manager.run_command(f'sudo cp {INTERNAL_CERT} {SAML_CERT}')
+    manager.run_command(f'sudo cp -a {INTERNAL_CERT} {SAML_CERT}')
     manager.run_command('sudo supervisorctl restart cloudify-restservice')
     logger.info('Waiting for manager service to finish restarting.')
     manager.wait_for_manager()

cosmo_tester/test_suites/bootstrap/teardown_test.py

@@ -1,3 +1,5 @@
+import json
+
 from cosmo_tester.framework.examples import get_example_deployment
 from cosmo_tester.framework.util import substitute_testing_version
 
@@ -42,17 +44,14 @@ def _test_teardown(function_scoped_manager, ssh_key, logger, test_config):
     # When the example deployment installs the agent this group will exist.
     # It shouldn't be deleted afterwards in case files on the system are in
     # that group.
+    # We also don't remove the agents dir for possible subsequent installs
     expected_diffs['os groups'] = {'cfyagent'}
+    expected_diffs['folders in /opt'] = {'cloudify-agent'}
 
-    function_scoped_manager.teardown(kill_certs=False)
+    assert json.loads(function_scoped_manager.run_command(
+        'cfy agents list -a --json').stdout) == []
 
-    # The agents dir should be empty, so let's remove it.
-    function_scoped_manager.run_command(
-        'rmdir /opt/cloudify-agent-{}'.format(
-            test_config['testing_version'].replace('-ga', '')
-        ),
-        use_sudo=True,
-    )
+    function_scoped_manager.teardown(kill_certs=False)
 
     current_state = _get_system_state(function_scoped_manager)
     diffs = {}

cosmo_tester/test_suites/cluster/broker_management_test.py

@@ -45,12 +45,12 @@ def add_to_hosts(target_broker, new_entry_brokers):
 
 def backup_hosts(brokers_list):
     for broker in brokers_list:
-        broker.run_command('sudo cp /etc/hosts /etc/hosts.bak')
+        broker.run_command('sudo cp -a /etc/hosts /etc/hosts.bak')
 
 
 def restore_hosts(brokers_list):
     for broker in brokers_list:
-        broker.run_command('sudo cp /etc/hosts.bak /etc/hosts')
+        broker.run_command('sudo cp -a /etc/hosts.bak /etc/hosts')
         broker.run_command('sudo rm /etc/hosts.bak')