diff --git a/README.md b/README.md index cf2c61eb9..60f3b2cfb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ pytest --pdb -s cosmo_tester/test_suites/general/simple_deployment_test.py::test --pdb is recommended for manual test runs as this will pause test execution on failure and may allow you to gain valuable insight into the cause of the failure. -s is recommended in order to ensure all test output is shown. +#### Tests which use an external DB +Some of our tests (currently only `cosmo_tester/test_suites/cluster/external_component_cluster_test.py::test_upgrade_external_db`) use an external database. +The FQDN and password of the database should be provided as environment variables `EXTDB_FQDN=...` and `EXTDB_PSWD=...` when running the test manually. +Developers may refer to the `pipelines-k8s/system-tests` pipeline in our `cloudify-build-system` repo for the values of our test-db on RDS. + + ## Using the config in tests There are two supported ways of accessing the config within tests. diff --git a/aws_cleanup b/aws_cleanup index 5e9a1833c..d82a43775 100755 --- a/aws_cleanup +++ b/aws_cleanup @@ -12,6 +12,7 @@ except ImportError: print('You need to pip install boto3==1.12.13') sys.exit(1) +TEST_GENERATED = '*pytest*' NOT_TERMINATED = [ 'pending', 'running', 'shutting-down', 'stopping', 'stopped' ] @@ -164,6 +165,16 @@ def get_vpcs(client, owner): ], )['Vpcs'] +def get_all_vpcs(client): + vpcs = [] + for vpc in client.describe_vpcs( + Filters = [ + {'Name' : 'tag:CreatedBy' , 'Values' : [TEST_GENERATED]} + ] + )['Vpcs']: + if not 'Name' in [ k['Key'] for k in vpc['Tags'] ]: + vpcs.append(vpc) + return vpcs def get_subnets(client, vpc): return client.describe_subnets( @@ -422,8 +433,32 @@ def delete_resources(client, resources_by_test): # investigate by hand if the script can't handle it. client.delete_vpc(VpcId=vpc) +def collect_resources_from_unknown_test(client, owner): + vpcs = get_all_vpcs(client) + for i, _ in enumerate(vpcs): + vpcs[i]['Tags'].append( { 'Key' : 'Name', 'Value' : 'unknowntest'+str(i) + "_" \ + + str(datetime.now().strftime('%Y%m%d%H%M%S'))} ) + resources_by_test = {} + for i, vpc in enumerate(vpcs): + vpc_id = vpc['VpcId'] -def main(older_than, newer_than, owner, delete): + instances = get_instances(client, vpc_id) + interfaces = get_interfaces(client, vpc_id) + subnets = get_subnets(client, vpc_id) + route_tables = get_route_tables(client, vpc_id) + security_groups = get_security_groups(client, vpc_id) + internet_gateways = get_internet_gateways(client, vpc_id) + + group_by_previous_tests(instances, interfaces, + subnets, route_tables, + internet_gateways, + security_groups, + vpc, resources_by_test) + + output_resources_by_test(resources_by_test) + return resources_by_test + +def main(older_than, newer_than, owner, delete, unknown): conf = load_config() client = get_ec2_client(conf) @@ -433,6 +468,7 @@ def main(older_than, newer_than, owner, delete): resources_by_test = {} vpcs = get_vpcs(client, owner) + for vpc in vpcs: vpc_id = vpc['VpcId'] instances = get_instances(client, vpc_id) @@ -441,7 +477,7 @@ def main(older_than, newer_than, owner, delete): route_tables = get_route_tables(client, vpc_id) security_groups = get_security_groups(client, vpc_id) internet_gateways = get_internet_gateways(client, vpc_id) - + group_by_previous_tests(instances, interfaces, subnets, route_tables, internet_gateways, @@ -453,13 +489,15 @@ def main(older_than, newer_than, owner, delete): resources_by_test) filter_test_groups(resources_by_test, older_than, newer_than) - output_resources_by_test(resources_by_test) - + if delete: print('Deleting') delete_resources(client, resources_by_test) - + if unknown: + print('Deleting resources from unkown tests') + unknown_resources = collect_resources_from_unknown_test(client, owner) + delete_resources(client, unknown_resources) if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -492,8 +530,13 @@ if __name__ == '__main__': action='store_true', default=False, ) + parser.add_argument( + '-u', '--unknown', + help='Delete discovered resources from unkown tests.', + action='store_true', + default=False, + ) args = parser.parse_args() - main(older_than=args.older, newer_than=args.newer, owner=args.owner, - delete=args.delete) + main(older_than=args.older, newer_than=args.newer, owner=args.owner, delete=args.delete, unknown=args.unknown) diff --git a/cosmo_tester/config_schemas/base.yaml b/cosmo_tester/config_schemas/base.yaml index 571ffdecd..47ad269ba 100644 --- a/cosmo_tester/config_schemas/base.yaml +++ b/cosmo_tester/config_schemas/base.yaml @@ -7,4 +7,4 @@ target_platform: valid_values: [openstack, aws] testing_version: description: Which manager version we're testing. Note that this is expected to be in the form -. Bad things may happen without the hyphen. - default: 7.0.0-.dev1 + default: 7.1.0-.dev1 diff --git a/cosmo_tester/config_schemas/cfy_cluster_manager.yaml b/cosmo_tester/config_schemas/cfy_cluster_manager.yaml index 70c00c866..e9f0a85e2 100644 --- a/cosmo_tester/config_schemas/cfy_cluster_manager.yaml +++ b/cosmo_tester/config_schemas/cfy_cluster_manager.yaml @@ -1,4 +1,5 @@ namespace: cfy_cluster_manager rpm_path: description: The cfy_cluster_manager RPM path to install. - default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.4/ga-release/cloudify-cluster-manager-1.1.4-ga.el7.x86_64.rpm' + default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.6/ga-release/cloudify-cluster-manager-1.1.6-ga.el7.x86_64.rpm' + diff --git a/cosmo_tester/config_schemas/platform_aws.yaml b/cosmo_tester/config_schemas/platform_aws.yaml index f1dfb7aaa..c172818ad 100644 --- a/cosmo_tester/config_schemas/platform_aws.yaml +++ b/cosmo_tester/config_schemas/platform_aws.yaml @@ -28,7 +28,7 @@ ubuntu_14_04_image: default: ami-005af4c3162f495fa ubuntu_16_04_image: description: Image to use for Ubuntu 16.04 on this platform. - default: ami-0a9aac550bc5711d3 + default: ami-0f29c8402f8cce65c ubuntu_18_04_image: description: Image to use for Ubuntu 18.04 on this platform. default: ami-02f0341ac93c96375 diff --git a/cosmo_tester/framework/examples.py b/cosmo_tester/framework/examples.py index b1ac7766f..921fc1201 100644 --- a/cosmo_tester/framework/examples.py +++ b/cosmo_tester/framework/examples.py @@ -144,9 +144,10 @@ def install(self): self.execute('install') self.installed = True - def uninstall(self, check_files_are_deleted=True, delete_dep=True): + def uninstall(self, check_files_are_deleted=True, delete_dep=True, + parameters=None): self.logger.info('Cleaning up example.') - self.execute('uninstall') + self.execute('uninstall', parameters=parameters) self.installed = False if check_files_are_deleted: self.check_all_test_files_deleted() diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 0c4839806..64baffd86 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -854,6 +854,10 @@ def download_rest_ca(self, force=False): '/etc/cloudify/ssl/cloudify_internal_ca_cert.pem', self.api_ca_path, ) + # close the current restclient session to force making a new connection + # using the new certificate, on first use after this call + if self.client: + self.client._client._session.close() @only_manager def clean_local_rest_ca(self): @@ -922,7 +926,8 @@ def _is_manager_image_type(self): def _is_rhel8_supported(self): if self.image_type == 'master': return True - if parse_version(self.image_type) >= parse_version('6.4.0'): + if parse_version(self.image_type.split('-')[0]) \ + >= parse_version('6.4.0'): return True return False @@ -986,6 +991,7 @@ def rsync_backup(self): 'Creating Rsync backup for host {}. Might take up to 5 ' 'minutes...'.format(self.deployment_id)) self.run_command("mkdir /cfy_backup", use_sudo=True) + self.run_command("chmod o+r /cfy_backup", use_sudo=True) rsync_backup_file = self._tmpdir / 'rsync_backup_{0}'.format( self.ip_address) locations = ' '.join(RSYNC_LOCATIONS) @@ -1134,7 +1140,7 @@ def __init__(self, else: self.server_flavor = self._test_config.platform['linux_size'] - def create(self): + def create(self, use_fqdn=False): """Creates the infrastructure for a Cloudify manager.""" self._logger.info('Creating image based cloudify instances: ' '[number_of_instances=%d]', len(self.instances)) @@ -1173,7 +1179,7 @@ def create(self): test_identifier, instance.is_manager, instance.image_type) - self._finish_deploy_test_vms() + self._finish_deploy_test_vms(use_fqdn=use_fqdn) for instance in self.instances: if instance.is_manager and not instance.bootstrappable: @@ -1565,7 +1571,7 @@ def _populate_aws_platform_properties(self): self._platform_resource_ids = resource_ids - def _finish_deploy_test_vms(self): + def _finish_deploy_test_vms(self, use_fqdn=False): node_instances = {} for vm_id, details in self._test_vm_installs.items(): execution, index = details @@ -1580,6 +1586,7 @@ def _finish_deploy_test_vms(self): self._update_instance( index, node_instance, + use_fqdn=use_fqdn, ) node_instances[index] = node_instance @@ -1607,11 +1614,16 @@ def _finish_undeploy_test_vms(self): util.delete_deployment(self._infra_client, vm_id, self._logger) - def _update_instance(self, server_index, node_instance): + def _update_instance(self, server_index, node_instance, use_fqdn=False): instance = self.instances[server_index] runtime_props = node_instance['runtime_properties'] public_ip_address = runtime_props['public_ip_address'] + if use_fqdn and self._test_config['target_platform'] == 'aws': + zone = runtime_props['resource']['Placement']['AvailabilityZone'] + public_ip_address = f"ec2-{public_ip_address.replace('.', '-')}."\ + f"{zone[:-1]}.compute.amazonaws.com" + private_ip_address = runtime_props['ipv6_address'] if self.ipv6_net \ else runtime_props['ip'] diff --git a/cosmo_tester/framework/util.py b/cosmo_tester/framework/util.py index 63f9a0a7c..58842f3bd 100644 --- a/cosmo_tester/framework/util.py +++ b/cosmo_tester/framework/util.py @@ -598,6 +598,8 @@ def delete_deployment(client, deployment_id, logger): for _ in range(40): found = False deployments = client.deployments.list() + if not deployments: + break for deployment in deployments: if deployment['id'] == deployment_id: found = True diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml index 6603be65e..b89d8906c 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml index f5071e01e..0dca5f9ff 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml index 5ed0dd8e1..04fc3ff7f 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml index 024f60613..94f984572 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - plugin:cloudify-openstack-plugin inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml index ac5bee496..4e82df3ae 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - plugin:cloudify-openstack-plugin inputs: diff --git a/cosmo_tester/resources/plugin/old_test_plugin.zip b/cosmo_tester/resources/plugin/old_test_plugin.zip deleted file mode 100644 index fd43096c9..000000000 Binary files a/cosmo_tester/resources/plugin/old_test_plugin.zip and /dev/null differ diff --git a/cosmo_tester/test_suites/agent/__init__.py b/cosmo_tester/test_suites/agent/__init__.py index 9c9318d16..0a54e1f3a 100644 --- a/cosmo_tester/test_suites/agent/__init__.py +++ b/cosmo_tester/test_suites/agent/__init__.py @@ -1,3 +1,5 @@ +from cloudify.models_states import AgentState + from cosmo_tester.framework.test_hosts import Hosts, VM @@ -25,8 +27,13 @@ def get_test_prerequisites(ssh_key, module_tmpdir, test_config, logger, def validate_agent(manager, example, test_config, broken_system=False, install_method='remote', upgrade=False): - agents = list(manager.client.agents.list(tenant_name=example.tenant, - _all_tenants=True)) + agents = list( + manager.client.agents.list( + tenant_name=example.tenant, + state=AgentState.STARTED, + _all_tenants=True, + ) + ) instances = list( manager.client.node_instances.list( tenant_name=example.tenant, node_id='vm', diff --git a/cosmo_tester/test_suites/auth/test_openldap.py b/cosmo_tester/test_suites/auth/test_openldap.py index 45c8b62e2..87df9e4a0 100644 --- a/cosmo_tester/test_suites/auth/test_openldap.py +++ b/cosmo_tester/test_suites/auth/test_openldap.py @@ -243,6 +243,13 @@ def _configure_openldap(host, logger): replace: olcTLSCACertificateFile olcTLSCACertificateFile: /etc/ssl/certs/ldapcacert.crt''', ) + host.run_command( + 'chgrp ldap' + ' /etc/openldap/certs/ldap-key.pem' + ' /etc/openldap/certs/ldap-cert.pem' + ' /etc/ssl/certs/ldapcacert.crt', + use_sudo=True, + ) host.run_command( 'ldapmodify -Y EXTERNAL -H ldapi:/// -f /tmp/certs.ldif', use_sudo=True, diff --git a/cosmo_tester/test_suites/auth/test_saml.py b/cosmo_tester/test_suites/auth/test_saml.py index 30a1006d1..cc526a8da 100644 --- a/cosmo_tester/test_suites/auth/test_saml.py +++ b/cosmo_tester/test_suites/auth/test_saml.py @@ -61,7 +61,7 @@ def test_saml_auth(image_based_manager, logger): def _activate_saml_auth(manager, logger): logger.info('Putting SAML cert in location and restarting rest service') - manager.run_command(f'sudo cp {INTERNAL_CERT} {SAML_CERT}') + manager.run_command(f'sudo cp -a {INTERNAL_CERT} {SAML_CERT}') manager.run_command('sudo supervisorctl restart cloudify-restservice') logger.info('Waiting for manager service to finish restarting.') manager.wait_for_manager() diff --git a/cosmo_tester/test_suites/bootstrap/teardown_test.py b/cosmo_tester/test_suites/bootstrap/teardown_test.py index 967e92a1a..d525f6f33 100644 --- a/cosmo_tester/test_suites/bootstrap/teardown_test.py +++ b/cosmo_tester/test_suites/bootstrap/teardown_test.py @@ -1,3 +1,5 @@ +import json + from cosmo_tester.framework.examples import get_example_deployment from cosmo_tester.framework.util import substitute_testing_version @@ -42,17 +44,14 @@ def _test_teardown(function_scoped_manager, ssh_key, logger, test_config): # When the example deployment installs the agent this group will exist. # It shouldn't be deleted afterwards in case files on the system are in # that group. + # We also don't remove the agents dir for possible subsequent installs expected_diffs['os groups'] = {'cfyagent'} + expected_diffs['folders in /opt'] = {'cloudify-agent'} - function_scoped_manager.teardown(kill_certs=False) + assert json.loads(function_scoped_manager.run_command( + 'cfy agents list -a --json').stdout) == [] - # The agents dir should be empty, so let's remove it. - function_scoped_manager.run_command( - 'rmdir /opt/cloudify-agent-{}'.format( - test_config['testing_version'].replace('-ga', '') - ), - use_sudo=True, - ) + function_scoped_manager.teardown(kill_certs=False) current_state = _get_system_state(function_scoped_manager) diffs = {} diff --git a/cosmo_tester/test_suites/cluster/broker_management_test.py b/cosmo_tester/test_suites/cluster/broker_management_test.py index f9af93001..a5d9c1a7c 100644 --- a/cosmo_tester/test_suites/cluster/broker_management_test.py +++ b/cosmo_tester/test_suites/cluster/broker_management_test.py @@ -45,12 +45,12 @@ def add_to_hosts(target_broker, new_entry_brokers): def backup_hosts(brokers_list): for broker in brokers_list: - broker.run_command('sudo cp /etc/hosts /etc/hosts.bak') + broker.run_command('sudo cp -a /etc/hosts /etc/hosts.bak') def restore_hosts(brokers_list): for broker in brokers_list: - broker.run_command('sudo cp /etc/hosts.bak /etc/hosts') + broker.run_command('sudo cp -a /etc/hosts.bak /etc/hosts') broker.run_command('sudo rm /etc/hosts.bak') diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml new file mode 100644 index 000000000..7635a47fa --- /dev/null +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml @@ -0,0 +1,75 @@ +ssh_key_path: '' # Filled by the test +ssh_user: '' # Filled by the test +cloudify_license_path: '' # Filled by the test +manager_rpm_path: '' # Filled by the test + +ldap: + server: '' + domain: '' + is_active_directory: '' + ca_cert: '' + username: '' + password: '' + dn_extra: '' + +ca_cert_path: '' + +load_balancer_ip: '' + +existing_vms: + node-1: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + + node-2: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + + node-3: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + +external_db_configuration: + host: '' + ca_path: '' + server_db_name: postgres + server_username: postgres + server_password: '' + cloudify_db_name: cloudify_db + cloudify_username: cloudify + cloudify_password: cloudify + +credentials: + manager: + admin_username: 'admin' + admin_password: 'admin' + + rabbitmq: + username: '' + password: '' + erlang_cookie: '' + + prometheus: + username: '' + password: '' diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml index bec1b2fd3..dcf3f37a3 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml @@ -39,14 +39,11 @@ validations: ssl_inputs: - external_cert_path: {{ node.cert_path }} - external_key_path: {{ node.key_path }} internal_cert_path: {{ node.cert_path }} internal_key_path: {{ node.key_path }} postgresql_client_cert_path: {{ node.cert_path }} postgresql_client_key_path: {{ node.key_path }} ca_cert_path: {{ ca_path }} - external_ca_cert_path: {{ ca_path }} prometheus: credentials: diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py index 4e53664e6..1b6fbf5a6 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py @@ -6,6 +6,7 @@ import yaml from cosmo_tester.framework import util +from cosmo_tester.test_suites.snapshots import upgrade_agents CLUSTER_MANAGER_RESOURCES_PATH = pkg_resources.resource_filename( @@ -29,7 +30,7 @@ def _update_three_nodes_config_dict_vms(config_dict, existing_vms_list): for i, node in enumerate(existing_vms_list, start=1): config_dict['existing_vms']['node-{0}'.format(i)].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': str(node.ip_address) + 'public_ip': f'{node.ip_address}' }) @@ -45,7 +46,7 @@ def _update_nine_nodes_config_dict_vms(config_dict, existing_vms_list): config_dict['existing_vms'][node_name].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': str(node.ip_address) + 'public_ip': f'{node.ip_address}' }) @@ -149,8 +150,52 @@ def _cluster_upgrade_test(test_config, base_version, nodes, 'echo -e \\\\nservice_management: systemd ' '| sudo tee -a {}'.format(conf) ) + _install_deployment_with_agent(manager, test_config, ssh_key, logger) _upgrade_cluster(nodes_list, manager, test_config, logger) + upgrade_agents(manager, logger, test_config) + + +def _install_deployment_with_agent(manager, test_config, ssh_key, logger): + dep_name = 'example' + + # copy test plugin + example blueprint + manager.run_command('yum install -y unzip', use_sudo=True) + manager.put_remote_file( + '/tmp/test_plugin.zip', + util.get_resource_path('plugin/test_plugin-1.0.0.zip'), + ) + manager.run_command('mkdir -p /tmp/example_bp') + manager.put_remote_file( + f'/tmp/{dep_name}_bp/{dep_name}.yaml', + util.get_resource_path('blueprints/compute/example.yaml'), + ) + manager.run_command('unzip /tmp/test_plugin.zip -d /tmp/test_plugin') + + # upload test plugin + blueprint + manager.run_command( + 'cfy plugins upload /tmp/test_plugin/*.wgn ' + '-y /tmp/test_plugin/plugin.yaml' + ) + manager.run_command( + f'cfy blueprints upload /tmp/{dep_name}_bp/{dep_name}.yaml' + f' -b {dep_name}' + ) + + # create and install deployment + inputs = { + 'server_ip': manager.ip_address, + 'path': f'/home/{manager.username}/test_file', + 'content': 'Test', + 'agent_user': manager.username + } + with open(ssh_key.private_key_path) as key_handle: + ssh_key_string = key_handle.read() + manager.run_command(f'cfy secrets create agent_key -s "{ssh_key_string}"') + manager.run_command( + f'cfy deployments create {dep_name} -b {dep_name} -i "{inputs}"') + manager.run_command( + f'cfy execution start install -d {dep_name}') def _get_config_dict(node_count, test_config, vm_user): diff --git a/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py b/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py index 07ef7d8ee..928a2e54d 100644 --- a/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py +++ b/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py @@ -7,8 +7,8 @@ @pytest.mark.three_vms @pytest.mark.upgrade @pytest.mark.parametrize('base_version', SUPPORTED_FOR_RPM_UPGRADE) -def test_three_nodes_cluster_upgrade(base_version, three_vms, test_config, - ssh_key, logger): +def test_three_nodes_cluster_upgrade(base_version, three_vms_fqdns, + test_config, ssh_key, logger): """Tests the command cfy_cluster_manager upgrade on a 3 nodes cluster.""" - _cluster_upgrade_test(test_config, base_version, three_vms, ssh_key, + _cluster_upgrade_test(test_config, base_version, three_vms_fqdns, ssh_key, logger) diff --git a/cosmo_tester/test_suites/cluster/conftest.py b/cosmo_tester/test_suites/cluster/conftest.py index 5ec867586..07c2fb4e8 100644 --- a/cosmo_tester/test_suites/cluster/conftest.py +++ b/cosmo_tester/test_suites/cluster/conftest.py @@ -34,6 +34,21 @@ def three_session_vms(request, ssh_key, session_tmpdir, test_config, hosts.destroy() +@pytest.fixture(scope='session') +def three_session_vms_fqdns(request, ssh_key, session_tmpdir, test_config, + session_logger): + hosts = Hosts(ssh_key, session_tmpdir, test_config, + session_logger, request, bootstrappable=True, + number_of_instances=3) + try: + hosts.create(use_fqdn=True) + if len(request.session.items) > 1: + hosts.rsync_backup() + yield hosts.instances + finally: + hosts.destroy() + + @pytest.fixture(scope='session') def three_ipv6_session_vms(request, ssh_key, session_tmpdir, test_config, session_logger): @@ -94,6 +109,21 @@ def nine_session_vms(request, ssh_key, session_tmpdir, test_config, hosts.destroy() +@pytest.fixture(scope='session') +def nine_session_vms_fqdns(request, ssh_key, session_tmpdir, test_config, + session_logger): + hosts = Hosts(ssh_key, session_tmpdir, test_config, + session_logger, request, bootstrappable=True, + number_of_instances=9) + try: + hosts.create(use_fqdn=True) + if len(request.session.items) > 1: + hosts.rsync_backup() + yield hosts.instances + finally: + hosts.destroy() + + @pytest.fixture(scope='function') def brokers(three_session_vms, test_config, logger, request): util.reboot_if_required(three_session_vms) @@ -237,6 +267,17 @@ def three_vms(three_session_vms, test_config, logger, request): util.rsync_restore(three_session_vms) +@pytest.fixture(scope='function') +def three_vms_fqdns(three_session_vms_fqdns, test_config, logger, request): + util.reboot_if_required(three_session_vms_fqdns) + for vm in three_session_vms_fqdns: + _ensure_installer_not_installed(vm) + yield _get_hosts(three_session_vms_fqdns, test_config, logger, + three_nodes_cluster=True, bootstrap=False) + if len(request.session.items) > 1: + util.rsync_restore(three_session_vms_fqdns) + + @pytest.fixture(scope='function') def three_vms_ipv6(three_ipv6_session_vms, test_config, logger, request): util.reboot_if_required(three_ipv6_session_vms) @@ -260,6 +301,18 @@ def nine_vms(nine_session_vms, test_config, logger, request): util.rsync_restore(nine_session_vms) +@pytest.fixture(scope='function') +def nine_vms_fqdns(nine_session_vms_fqdns, test_config, logger, request): + util.reboot_if_required(nine_session_vms_fqdns) + for vm in nine_session_vms_fqdns: + _ensure_installer_not_installed(vm) + yield _get_hosts(nine_session_vms_fqdns, test_config, logger, + broker_count=3, db_count=3, + manager_count=3, bootstrap=False) + if len(request.session.items) > 1: + util.rsync_restore(nine_session_vms_fqdns) + + def _ensure_installer_not_installed(vm): vm.wait_for_ssh() vm.run_command( @@ -716,12 +769,9 @@ def _bootstrap_manager_node(node, mgr_num, dbs, brokers, skip_bootstrap_list, if high_security: node.install_config['ssl_inputs'] = { - 'external_cert_path': node.remote_cert, - 'external_key_path': node.remote_key, 'internal_cert_path': node.remote_cert, 'internal_key_path': node.remote_key, 'ca_cert_path': node.remote_ca, - 'external_ca_cert_path': node.remote_ca, } node.install_config['manager']['security'][ 'ssl_enabled'] = True diff --git a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py index f6c489362..2a2408b18 100644 --- a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py +++ b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py @@ -1,3 +1,4 @@ +import os import pytest from cosmo_tester.test_suites.cluster import check_managers @@ -6,6 +7,12 @@ create_snapshot, restore_snapshot, ) +from .cfy_cluster_manager_shared import ( + _get_config_dict, + _set_rpm_path, + _install_cluster, + _upgrade_cluster, +) # This is to confirm that we work with a single DB endpoint set (e.g. on a @@ -32,3 +39,66 @@ def test_cluster_single_db(cluster_with_single_db, logger, ssh_key, admin_password=mgr1.mgr_password) check_managers(mgr1, mgr2, example) + + +@pytest.mark.three_vms +@pytest.mark.upgrade +@pytest.mark.parametrize('base_version', ['6.3.2-ga', '6.4.1-ga']) +def test_upgrade_external_db( + base_version, three_vms, logger, ssh_key, test_config): + ext_db_fqdn = os.environ["EXTDB_FQDN"] + ext_db_password = os.environ["EXTDB_PSWD"] + + nodes_list = [node for node in three_vms] + mgr = nodes_list[0] + + _cleanup_external_db(ext_db_fqdn, ext_db_password, mgr) + + # download the ca-cert bundle for the external db + mgr.run_command('curl https://truststore.pki.rds.amazonaws.com/' + 'eu-west-1/eu-west-1-bundle.pem ' + '-o /tmp/eu-west-1-bundle.pem') + + config_dict = _get_config_dict('3+ext_db', test_config, mgr.username) + _set_rpm_path(config_dict, test_config, base_version) + _update_3_nodes_ext_db_config_dict_vms( + config_dict, nodes_list, ext_db_fqdn, ext_db_password) + _install_cluster(mgr, three_vms, config_dict, test_config, + ssh_key, logger) + _upgrade_cluster(nodes_list, mgr, test_config, logger) + + +def _cleanup_external_db(ext_db_fqdn, ext_db_password, node): + cleanup_script = f''' +if PGPASSWORD={ext_db_password} psql -U postgres -h {ext_db_fqdn} -p 5432 \ + -c "\\l" | grep -q "cloudify"; then + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database cloudify_db" + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database stage" + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database composer" + PGPASSWORD={ext_db_password} psql -U postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop user cloudify" +fi + ''' + node.run_command('yum install -y postgresql', use_sudo=True) + node.run_command(cleanup_script) + node.run_command('yum remove -y postgresql', use_sudo=True) + + +def _update_3_nodes_ext_db_config_dict_vms( + config_dict, existing_vms_list, ext_db_fqdn, ext_db_password): + for i, node in enumerate(existing_vms_list, start=1): + config_dict['existing_vms']['node-{0}'.format(i)].update({ + 'private_ip': str(node.private_ip_address), + 'public_ip': f'{node.private_ip_address}' + }) + # Just put some FQDN, because when public_ip is an IP address, + # the external certificate cannot be provided. + # Public IPs are not used in this test. + config_dict['external_db_configuration']['host'] = ext_db_fqdn + config_dict['external_db_configuration']['ca_path'] = \ + '/tmp/eu-west-1-bundle.pem' + config_dict['external_db_configuration']['server_password'] = \ + ext_db_password diff --git a/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py b/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py index 8cc2d1321..98da1635f 100644 --- a/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py +++ b/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py @@ -8,8 +8,8 @@ @pytest.mark.nine_vms @pytest.mark.upgrade @pytest.mark.parametrize('base_version', SUPPORTED_FOR_RPM_UPGRADE) -def test_nine_nodes_cluster_upgrade(base_version, nine_vms, test_config, +def test_nine_nodes_cluster_upgrade(base_version, nine_vms_fqdns, test_config, ssh_key, logger): """Tests the command cfy_cluster_manager upgrade on a 9 nodes cluster.""" - _cluster_upgrade_test(test_config, base_version, nine_vms, + _cluster_upgrade_test(test_config, base_version, nine_vms_fqdns, ssh_key, logger) diff --git a/cosmo_tester/test_suites/general/cfy_manager_test.py b/cosmo_tester/test_suites/general/cfy_manager_test.py index ab3ad7550..e96e6f8a3 100644 --- a/cosmo_tester/test_suites/general/cfy_manager_test.py +++ b/cosmo_tester/test_suites/general/cfy_manager_test.py @@ -167,7 +167,14 @@ def test_cfy_manager_upgrade(base_manager, ssh_key, logger, test_config): upgrade_agents(base_manager, logger, test_config) validate_cluster_status_and_agents(base_manager, example.tenant, logger) - example.uninstall() + # We'd like to ignore a failure of file.delete due to a possible old + # version of the test plugin + if base_manager.image_type.startswith('5') or \ + base_manager.image_type.startswith('6'): + example.uninstall(parameters={'ignore_failure': True}, + check_files_are_deleted=False) + else: + example.uninstall() def _edit_security_config(manager): diff --git a/cosmo_tester/test_suites/general/status_test.py b/cosmo_tester/test_suites/general/status_test.py index c529b331d..427e6267f 100644 --- a/cosmo_tester/test_suites/general/status_test.py +++ b/cosmo_tester/test_suites/general/status_test.py @@ -24,7 +24,7 @@ def test_status(image_based_manager, logger): # Allow time for services to start when we restart them -@retrying.retry(stop_max_attempt_number=10, wait_fixed=1000) +@retrying.retry(stop_max_attempt_number=20, wait_fixed=1000) def _check_status(manager, logger, healthy=True): if healthy: logger.info('Checking for healthy status') diff --git a/cosmo_tester/test_suites/multi_net/multi_network_test.py b/cosmo_tester/test_suites/multi_net/multi_network_test.py index 50765d051..8ca01457f 100644 --- a/cosmo_tester/test_suites/multi_net/multi_network_test.py +++ b/cosmo_tester/test_suites/multi_net/multi_network_test.py @@ -213,7 +213,7 @@ def proxy_prepare_hosts(instances, logger): # setup the proxy - simple socat services that forward all TCP connections # to the manager proxy.run_command('yum install socat -y', use_sudo=True) - for port in [5671, 53333, 15671]: + for port in [443, 5671, 15671]: service = 'proxy_{0}'.format(port) filename = '/usr/lib/systemd/system/{0}.service'.format(service) logger.info('Deploying proxy service file') @@ -242,18 +242,16 @@ def test_agent_via_proxy(proxy_hosts, # to make sure that the agents go through the proxy, and not connect to # the manager directly, we block all communication on the manager's - # rabbitmq and internal REST endpoint, except from the proxy (and from - # localhost) + # rabbitmq endpoint, except from the proxy (and from localhost) manager_ip = manager.private_ip_address proxy_ip = proxy.private_ip_address - for port in [5671, 53333]: + manager.run_command( + 'iptables -I INPUT -p tcp -s 0.0.0.0/0 --dport 5671 -j DROP', + use_sudo=True) + for ip in [proxy_ip, manager_ip, '127.0.0.1']: manager.run_command( - 'iptables -I INPUT -p tcp -s 0.0.0.0/0 --dport {0} -j DROP' - .format(port), use_sudo=True) - for ip in [proxy_ip, manager_ip, '127.0.0.1']: - manager.run_command( - 'iptables -I INPUT -p tcp -s {0} --dport {1} -j ACCEPT' - .format(ip, port), use_sudo=True) + f'iptables -I INPUT -p tcp -s {ip} --dport 5671 -j ACCEPT', + use_sudo=True) example = get_example_deployment( manager, ssh_key, logger, 'agent_via_proxy', test_config, vm) diff --git a/cosmo_tester/test_suites/snapshots/inplace_restore_test.py b/cosmo_tester/test_suites/snapshots/inplace_restore_test.py index c04d923f8..5cd6c90aa 100644 --- a/cosmo_tester/test_suites/snapshots/inplace_restore_test.py +++ b/cosmo_tester/test_suites/snapshots/inplace_restore_test.py @@ -80,13 +80,9 @@ def test_inplace_restore(manager_and_vm, 'key_path': '/tmp/ssl_backup/monitoring_key.pem', } manager.install_config['ssl_inputs'] = { - 'external_cert_path': '/tmp/ssl_backup/cloudify_external_cert.pem', - 'external_key_path': '/tmp/ssl_backup/cloudify_external_key.pem', 'internal_cert_path': '/tmp/ssl_backup/cloudify_internal_cert.pem', 'internal_key_path': '/tmp/ssl_backup/cloudify_internal_key.pem', 'ca_cert_path': '/tmp/ssl_backup/cloudify_internal_ca_cert.pem', - 'external_ca_cert_path': - '/tmp/ssl_backup/cloudify_internal_ca_cert.pem', } manager.bootstrap() upload_snapshot(manager, snapshot_path, snapshot_name, logger) diff --git a/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py b/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py index ce3500992..fe35531f3 100644 --- a/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py +++ b/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py @@ -213,7 +213,7 @@ def prepare_old_manager_resources(manager, logger, ssh_key, test_config, 'cloudify-windows-agent.exe' ) manager.run_command('curl -Lo {} {}'.format(tmp_path, agent_url)) - manager.run_command('sudo cp {} {}'.format( + manager.run_command('sudo cp -a {} {}'.format( tmp_path, agent_destination)) for tenant in INSTALL_TENANTS: diff --git a/cosmo_tester/test_suites/summary/summary_error_test.py b/cosmo_tester/test_suites/summary/summary_error_test.py index fc1f9f880..430fdf97c 100644 --- a/cosmo_tester/test_suites/summary/summary_error_test.py +++ b/cosmo_tester/test_suites/summary/summary_error_test.py @@ -57,7 +57,7 @@ def test_bad_field_selected(prepared_manager, summary_type): '{endpoint}.'.format(endpoint=summary_type) ) except CloudifyClientError as err: - components = ['notafield', 'not', 'summary', 'Valid'] + components = ['notafield', 'summary', 'Invalid'] components.extend(SUPPORTED_FIELDS[summary_type]) if not all(component in str(err) for component in components): raise AssertionError( @@ -96,7 +96,7 @@ def test_bad_subfield_selected(prepared_manager, summary_type): '{endpoint}.'.format(endpoint=summary_type) ) except CloudifyClientError as err: - components = ['notafield', 'not', 'summary', 'Valid'] + components = ['notafield', 'summary', 'Invalid'] components.extend(SUPPORTED_FIELDS[summary_type]) if not all(component in str(err) for component in components): raise AssertionError( diff --git a/requirements.txt b/requirements.txt index cdd09b836..bf44de57a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ charset-normalizer==2.1.1 # via # aiohttp # requests -cloudify-common @ https://github.com/cloudify-cosmo/cloudify-common/archive/7.0.0-build.zip +cloudify-common @ https://github.com/cloudify-cosmo/cloudify-common/archive/master.zip # via -r requirements.in cryptography==38.0.4 # via diff --git a/setup.py b/setup.py index 5df8e99b1..17317c110 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='cloudify-system-tests', - version='7.0.0', + version='7.1.0.dev1', author='Cloudify', author_email='cosmo-admin@cloudify.co', packages=['cosmo_tester'],