From 424ce9f356ae5bc34b1269785ecd18e49958ad52 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo <71768+gionn@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:08:14 +0100 Subject: [PATCH] OPSEXP-2450 Search Enterprise as default search engine (#783) --- .envrc | 2 +- .github/workflows/enteprise.yml | 2 +- .secrets.baseline | 4 +- docs/README.md | 6 +-- docs/deployment-guide.md | 11 ++--- docs/playbook-upgrade.md | 9 +++++ inventory_ha.yml | 37 ++++++++++++----- inventory_local.yml | 40 ++++++++++++++----- inventory_ssh.yml | 28 ++++++++++--- molecule/default/verify.yml | 15 ++++--- molecule/elasticsearch/converge.yml | 7 ---- ...-integration-instance.yml => instance.yml} | 0 molecule/elasticsearch/molecule.yml | 8 +++- molecule/multimachine/molecule.yml | 2 +- molecule/opensearch/molecule.yml | 1 - .../{elasticsearch => opensearch}/verify.yml | 22 +++++----- playbooks/acs.yml | 24 ++--------- playbooks/prerun-checks.yml | 16 ++++++++ playbooks/secrets.yml | 4 +- requirements.yml | 6 +-- .../elasticsearch/molecule/default/verify.yml | 6 ++- roles/elasticsearch/tasks/main.yml | 15 ++++--- scripts/vagrant_provision.sh | 2 +- 23 files changed, 166 insertions(+), 101 deletions(-) delete mode 100644 molecule/elasticsearch/converge.yml rename molecule/elasticsearch/host_vars/{search-enteprise-integration-instance.yml => instance.yml} (100%) rename molecule/{elasticsearch => opensearch}/verify.yml (81%) diff --git a/.envrc b/.envrc index 64b8f13d6..86b658b02 100644 --- a/.envrc +++ b/.envrc @@ -4,7 +4,7 @@ export AWS_REGION=eu-west-1 export MOLECULE_IT_AWS_VPC_SUBNET_ID=subnet-6bdd4223 export BRANCH_NAME=local export BUILD_NUMBER=1 -export DTAS_VERSION=v1.2.0 +export DTAS_VERSION=v1.5.3 export MOLECULE_IT_ID=$(echo "$LOGNAME" | sha256sum | cut -c1-6) ANSIBLE_VAULT_PASSWORD_FILE=$(expand_path ./.vault_pass.txt) export ANSIBLE_VAULT_PASSWORD_FILE diff --git a/.github/workflows/enteprise.yml b/.github/workflows/enteprise.yml index 496d99e39..2862b8833 100644 --- a/.github/workflows/enteprise.yml +++ b/.github/workflows/enteprise.yml @@ -10,7 +10,7 @@ on: - .pre-commit-config.yaml env: - DTAS_VERSION: v1.2.2 + DTAS_VERSION: v1.5.3 BUILD_NUMBER: ${{ github.run_id }} PY_COLORS: 1 PYTHONUNBUFFERED: 1 diff --git a/.secrets.baseline b/.secrets.baseline index 3e6c67da5..40351dbc8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -144,7 +144,7 @@ "filename": "playbooks/acs.yml", "hashed_secret": "3a0b8a438a9efa61267357269709a946d797b9bd", "is_verified": false, - "line_number": 407, + "line_number": 391, "is_secret": false } ], @@ -259,5 +259,5 @@ } ] }, - "generated_at": "2024-01-17T10:11:58Z" + "generated_at": "2024-02-22T16:12:32Z" } diff --git a/docs/README.md b/docs/README.md index 9976bc3f2..8133a8493 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,9 +45,9 @@ The playbook uses the following roles: * **nginx** - deploys and configures Nginx as a proxy * **postgres** - deploys and configures PostgreSQL * **repository** - deploys and configures Alfresco Repository and Alfresco Share -* **search** - deploys and configures Alfresco Search Services -* **search_enteprise** - deploys and configures Alfresco Search Enterprise (as - an alternative to Alfresco Search Services) +* **search** - deploys and configures Alfresco Search Services (as + an alternative to Search Enterprise) +* **search_enterprise** - deploys and configures Alfresco Search Enterprise * **sfs** - deploys and configures Alfresco Shared File Store * **sync** - deploys and configures Alfresco Sync Service * **tomcat** - deploys and configures Apache Tomcat diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 70e8c951c..8c488b944 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -12,6 +12,7 @@ If it's your first time with Ansible, please have a read at [Ansible concepts](h * [Getting started](#getting-started) * [Get the playbook](#get-the-playbook) * [Setup runtime environment](#setup-runtime-environment) + * [Minimal configuration](#minimal-configuration) * [Understanding the playbook](#understanding-the-playbook) * [The control node](#the-control-node) * [Understanding the inventory file](#understanding-the-inventory-file) @@ -274,15 +275,15 @@ An ACS inventory file has the following groups a host can belong to: * `external_activemq`: an alternative group to `activemq` in case you don't want to deploy ActiveMQ using our basic activemq role but instead use an ActiveMQ instance of yours which matches your hosting standards. -* `search`: a single host on which to deploy Alfresco Search services. -* `search_enterprise`: one or more hosts on which deploy Search Enterprise, as - an alternative to Alfresco Search. +* `search`: a single host on which to deploy Alfresco Search services, as an + alternative to Search Enterprise. +* `search_enterprise`: one or more hosts on which deploy Search Enterprise. * `elasticsearch`: one or more hosts on which deploy the ElasticSearch cluster backing Search Enterprise. * `external_elasticsearch`: an alternative group to `elasticsearch` in case you don't want to deploy ElasticSearch using the [community ElasticSearch - role](https://github.com/buluma/ansible-role-elasticsearch) but instead use an - ElasticSearch cluster of yours which matches your hosting standards. + role](https://github.com/geerlingguy/ansible-role-elasticsearch) but instead + use an ElasticSearch cluster of yours which matches your hosting standards. * `nginx`: a single host on which the playbook will deploy an NGINX reverse proxy configured for the numerous http based service in the platform. * `acc`: a single host where you want the Alfresco Control Center UI to be installed diff --git a/docs/playbook-upgrade.md b/docs/playbook-upgrade.md index bab0b3157..86aac761d 100644 --- a/docs/playbook-upgrade.md +++ b/docs/playbook-upgrade.md @@ -2,6 +2,15 @@ ## Unreleased version +### Search Enterprise is the new default search engine + +The example inventories have been updated to default to Search Enterprise / +ElasticSearch (`search_enterprise` and `elasticsearch` groups) as the preferred +search engine from Enterprise since ACS 23.1.1. + +Search Services are still supported as before by assigning hosts to the `search` +group. + ### Passing Alfresco global properties In previous version we provided an empty `alfresco-global.properties` file to diff --git a/inventory_ha.yml b/inventory_ha.yml index 398aab462..714e6e1dc 100644 --- a/inventory_ha.yml +++ b/inventory_ha.yml @@ -16,59 +16,76 @@ all: database: hosts: db.infra.local: + repository: hosts: ecm1.infra.local: ecm2.infra.local: ingester.infra.local: cluster_keepoff: true + activemq: hosts: mq.infra.local: + + # Solr search engine (alternative to Enterprise Search) search: hosts: - fts.infra.local: - # Enterprise Search section start - # alternative to `search` group + + # Enterprise Search (default search engine) search_enterprise: hosts: + fts.infra.local: elasticsearch: hosts: - # Enterprise Search section end + fts.infra.local: + + # Keycloak SSO support (optional) identity: hosts: + nginx: hosts: proxy.infra.local: + acc: hosts: web.infra.local: + adw: hosts: web.infra.local: + transformers: hosts: render.infra.local: + syncservice: hosts: sync.infra.local: + other_repo_clients: hosts: + + trusted_resource_consumers: + children: + repository: + nginx: + adw: + other_repo_clients: + external_activemq: hosts: + external_elasticsearch: hosts: + external_identity: hosts: + external: children: external_activemq: external_elasticsearch: external_identity: other_repo_clients: - trusted_resource_consumers: - children: - repository: - nginx: - adw: - other_repo_clients: diff --git a/inventory_local.yml b/inventory_local.yml index 43f22d845..1581fee41 100644 --- a/inventory_local.yml +++ b/inventory_local.yml @@ -2,60 +2,78 @@ all: vars: ansible_connection: local + children: repository: hosts: localhost: + database: children: repository: + activemq: children: repository: + + # Solr search engine (alternative to Enterprise Search) search: children: - repository: - # Enterprise Search section start - # alternative to `search` group + + # Enterprise Search (default search engine) search_enterprise: children: + repository: elasticsearch: children: - # Enterprise Search section end + repository: + + # Keycloak SSO support (optional) identity: - hosts: + children: + nginx: children: repository: + acc: children: repository: + adw: children: repository: + transformers: children: repository: + syncservice: children: repository: + other_repo_clients: hosts: + + trusted_resource_consumers: + children: + repository: + nginx: + adw: + other_repo_clients: + external_activemq: hosts: + external_elasticsearch: hosts: + external_identity: hosts: + external: children: external_activemq: external_elasticsearch: external_identity: other_repo_clients: - trusted_resource_consumers: - children: - repository: - nginx: - adw: - other_repo_clients: diff --git a/inventory_ssh.yml b/inventory_ssh.yml index 3519f68ba..7ee400157 100644 --- a/inventory_ssh.yml +++ b/inventory_ssh.yml @@ -16,61 +16,79 @@ all: hosts: database_1: ansible_host: targetIP + repository: hosts: repository_1: ansible_host: targetIP + activemq: hosts: activemq_1: ansible_host: targetIP + + # Solr search engine (alternative to Enterprise Search) search: hosts: - search_1: - ansible_host: targetIP - # Enterprise Search section start - # alternative to `search` group + + # Enterprise Search (default search engine) search_enterprise: hosts: + search_1: + ansible_host: targetIP elasticsearch: hosts: - # Enterprise Search section end + search_1: + ansible_host: targetIP + + # Keycloak SSO support (optional) identity: hosts: + nginx: hosts: nginx_1: ansible_host: targetIP + acc: hosts: acc_1: ansible_host: targetIP + adw: hosts: adw_1: ansible_host: targetIP + transformers: hosts: transformers_1: ansible_host: targetIP + syncservice: hosts: syncservice_1: ansible_host: targetIP + other_repo_clients: hosts: + trusted_resource_consumers: children: repository: nginx: adw: other_repo_clients: + external_activemq: hosts: + external_elasticsearch: hosts: + external_identity: hosts: + external: children: external_activemq: diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index b8daae533..945cfba0a 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -30,19 +30,24 @@ replace: "https://{{ node_hostname }}" - name: Run pytest - vars: - selfsigned_cert_path: "{{ project_dir }}/configuration_files/ssl_certificates/{{ node_hostname }}.crt" + environment: + REQUESTS_CA_BUNDLE: "{{ project_dir }}/configuration_files/ssl_certificates/{{ node_hostname }}.crt" ansible.builtin.shell: chdir: "{{ dtas_dir }}" cmd: | - export REQUESTS_CA_BUNDLE={{ selfsigned_cert_path }} pytest --tb=line --color=no --configuration {{ test_config_file }} tests/ -s rescue: - name: Print multiline pytest stdout as best as we can debug: msg: "{{ ansible_failed_result.stdout_lines }}" - - name: Explicit failure after printing failure debug - fail: msg="Aborting due to pytest failure" + - name: Run pytest AGAIN only for failed tests + environment: + REQUESTS_CA_BUNDLE: "{{ project_dir }}/configuration_files/ssl_certificates/{{ node_hostname }}.crt" + ansible.builtin.shell: + chdir: "{{ dtas_dir }}" + cmd: | + pytest --tb=line --color=no --configuration {{ test_config_file }} tests/ -s --last-failed + - name: Verify adw plugins state hosts: adw diff --git a/molecule/elasticsearch/converge.yml b/molecule/elasticsearch/converge.yml deleted file mode 100644 index 716267486..000000000 --- a/molecule/elasticsearch/converge.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Run the playbook - ansible.builtin.import_playbook: ../../playbooks/acs.yml - vars: - autogen_unsecure_secrets: True - elasticsearch_cluster_initial_master_nodes: - - search-enteprise-integration-instance diff --git a/molecule/elasticsearch/host_vars/search-enteprise-integration-instance.yml b/molecule/elasticsearch/host_vars/instance.yml similarity index 100% rename from molecule/elasticsearch/host_vars/search-enteprise-integration-instance.yml rename to molecule/elasticsearch/host_vars/instance.yml diff --git a/molecule/elasticsearch/molecule.yml b/molecule/elasticsearch/molecule.yml index 5fa7fc25d..71a1e74fc 100644 --- a/molecule/elasticsearch/molecule.yml +++ b/molecule/elasticsearch/molecule.yml @@ -4,7 +4,7 @@ dependency: driver: name: docker platforms: - - name: search-enteprise-integration-instance + - name: instance image: $MOLECULE_ROLE_IMAGE dockerfile: ../../tests/molecule/Dockerfile-noprivs.j2 command: "/lib/systemd/systemd" @@ -22,10 +22,10 @@ platforms: - acc - adw - database - - elasticsearch - nginx - repository - search_enterprise + - elasticsearch - sfs - syncservice - transformers @@ -38,11 +38,15 @@ provisioner: ansible_args: - -e - "@../../tests/test-ssl.yml" + - -e + - autogen_unsecure_secrets=true inventory: links: group_vars: ../../group_vars host_vars: host_vars playbooks: prepare: ../default/prepare.yml + converge: ../../playbooks/acs.yml + verify: ../multimachine/verify.yml verifier: name: ansible diff --git a/molecule/multimachine/molecule.yml b/molecule/multimachine/molecule.yml index 96a6fbe2b..4f928b384 100644 --- a/molecule/multimachine/molecule.yml +++ b/molecule/multimachine/molecule.yml @@ -147,5 +147,5 @@ provisioner: playbooks: create: ../default/create.yml converge: ../../playbooks/acs.yml - verify: ../default/verify.yml destroy: ../default/destroy.yml + verify: ../default/verify.yml diff --git a/molecule/opensearch/molecule.yml b/molecule/opensearch/molecule.yml index c404ae024..8adb3738f 100644 --- a/molecule/opensearch/molecule.yml +++ b/molecule/opensearch/molecule.yml @@ -41,4 +41,3 @@ provisioner: create: ../default/create.yml converge: ../../playbooks/acs.yml destroy: ../default/destroy.yml - verify: ../default/verify.yml diff --git a/molecule/elasticsearch/verify.yml b/molecule/opensearch/verify.yml similarity index 81% rename from molecule/elasticsearch/verify.yml rename to molecule/opensearch/verify.yml index f4760b39a..dd0419f4f 100644 --- a/molecule/elasticsearch/verify.yml +++ b/molecule/opensearch/verify.yml @@ -1,4 +1,5 @@ -- name: Verify +--- +- name: Verify search enterprise hosts: search_enterprise gather_facts: false vars: @@ -22,7 +23,7 @@ until: error_log.stdout | regex_search('Started LiveIndexingApp') - name: Verify Reindex before first run - hosts: search_enterprise + hosts: search_enterprise[0] gather_facts: false vars: job_name: elasticsearch-connector-reindex.service @@ -32,16 +33,11 @@ that: - ansible_facts.services[job_name] is defined - ansible_facts.services[job_name].state == 'inactive' - -- name: Run the Elasticsearch Connector Reindex - import_playbook: ../../playbooks/search-enterprise-reindex.yml - -- name: Verify Reindex after first run - hosts: search_enterprise[0] - gather_facts: false - vars: - job_name: elasticsearch-connector-reindex.service - tasks: + - name: Run the Elasticsearch Connector Reindex + become: true + ansible.builtin.systemd: + state: started + name: elasticsearch-connector-reindex - name: "Check that {{ job_name }} is really up and running" become: true ansible.builtin.shell: journalctl -u {{ job_name }} @@ -50,5 +46,5 @@ delay: 10 until: error_log.stdout | regex_search('reindexByIds.*COMPLETED.*') -- name: Run default verify +- name: Default verify import_playbook: ../default/verify.yml diff --git a/playbooks/acs.yml b/playbooks/acs.yml index 412142818..f7adf4d8e 100644 --- a/playbooks/acs.yml +++ b/playbooks/acs.yml @@ -118,10 +118,6 @@ hosts: transformers gather_facts: false pre_tasks: - - name: Check wether we want mTLS for Tengines - ansible.builtin.set_fact: - tengine_mtls_required: >- - {{ groups.repository | difference(groups.transformers) | length > 0 }} - name: Build keystore role argument ansible.builtin.set_fact: transformers_keystore: @@ -133,7 +129,7 @@ pass: "{{ hostvars.localhost.certs_p12_passphrase }}" add_to_trusted_ca: true when: - - tengine_mtls_required + - ats_mtls_required - ats_mtls_capable roles: - role: "../roles/transformers" @@ -197,10 +193,6 @@ hosts: repository gather_facts: false pre_tasks: - - name: Check wether we want mTLS for Repostory - ansible.builtin.set_fact: - repo_mtls_required: >- - {{ groups.repository | difference(groups.transformers | default([])) | length > 0 }} - name: Build keystore role argument ansible.builtin.set_fact: repository_keystore: @@ -212,7 +204,7 @@ pass: "{{ hostvars.localhost.certs_p12_passphrase }}" add_to_trusted_ca: true when: - - repo_mtls_required + - ats_mtls_required - ats_mtls_capable roles: - role: "../roles/repository" @@ -242,10 +234,6 @@ hosts: transformers gather_facts: false pre_tasks: - - name: Check wether we want mTLS for Tengines - ansible.builtin.set_fact: - trouter_mtls_required: >- - {{ groups.repository | difference(groups.transformers) | length > 0 }} - name: Build keystore role argument ansible.builtin.set_fact: trouter_keystore: @@ -257,7 +245,7 @@ pass: "{{ hostvars.localhost.certs_p12_passphrase }}" add_to_trusted_ca: true when: - - trouter_mtls_required + - ats_mtls_required - ats_mtls_capable roles: - role: "../roles/trouter" @@ -284,10 +272,6 @@ hosts: transformers gather_facts: false pre_tasks: - - name: Check wether we want mTLS for Tengines - ansible.builtin.set_fact: - sfs_mtls_required: >- - {{ groups.repository | difference(groups.transformers) | length > 0 }} - name: Build keystore role argument ansible.builtin.set_fact: sfs_keystore: @@ -299,7 +283,7 @@ pass: "{{ hostvars.localhost.certs_p12_passphrase }}" add_to_trusted_ca: true when: - - sfs_mtls_required + - ats_mtls_required - ats_mtls_capable roles: - role: "../roles/sfs" diff --git a/playbooks/prerun-checks.yml b/playbooks/prerun-checks.yml index a0924b687..23d11a8b5 100644 --- a/playbooks/prerun-checks.yml +++ b/playbooks/prerun-checks.yml @@ -40,3 +40,19 @@ when: - known_urls | default([]) | length == 0 - cors.enabled | default(True) + +- name: Ensure mTLS requirements are met + hosts: repository:transformers + gather_facts: false + tasks: + - name: Check wether we want to enable mTLS for ATS deployment + ansible.builtin.set_fact: + ats_mtls_required: >- + {{ groups.repository | difference(groups.transformers | default([])) | length > 0 }} + + - name: Assert that search-enterprise group is empty when ATS mTLS is required + ansible.builtin.assert: + that: + - groups.search_enterprise | default([]) | length == 0 + fail_msg: mTLS enabled but not yet supported for search_enterprise + when: ats_mtls_required diff --git a/playbooks/secrets.yml b/playbooks/secrets.yml index b593c9ae3..90ddf6e8e 100644 --- a/playbooks/secrets.yml +++ b/playbooks/secrets.yml @@ -87,7 +87,7 @@ reposearch_shared_secret: "{{ hostvars.localhost.reposearch_shared_secret }}" - name: Set secrets for Repo database auth - hosts: repository:database + hosts: repository:database:search_enterprise gather_facts: false tasks: - name: Set repo_db_password secret @@ -103,7 +103,7 @@ sync_db_password: "{{ hostvars.localhost.sync_db_password }}" - name: Set secrets for ACtiveMQ auth - hosts: activemq:repository:transformers:syncservice + hosts: activemq:repository:transformers:syncservice:search_enterprise gather_facts: false tasks: - name: Set external activemq credentials if present diff --git a/requirements.yml b/requirements.yml index 03232f0a2..9b29e47a2 100644 --- a/requirements.yml +++ b/requirements.yml @@ -20,7 +20,5 @@ collections: version: 7.1.0 roles: - - name: buluma.elastic_repo - version: v0.4.0 - - name: buluma.elasticsearch - version: 1.2.0 + - name: geerlingguy.elasticsearch + version: 5.1.2 diff --git a/roles/elasticsearch/molecule/default/verify.yml b/roles/elasticsearch/molecule/default/verify.yml index b7dcf5caf..bc6812d5c 100644 --- a/roles/elasticsearch/molecule/default/verify.yml +++ b/roles/elasticsearch/molecule/default/verify.yml @@ -17,8 +17,12 @@ - tmp_path_stat.stat.uid == 0 - tmp_path_stat.stat.gid == 0 - - name: Check if port 9200/tcp is listening + - name: Gather facts + ansible.builtin.setup: + + - name: "Check if port 9200/tcp is listening on {{ ansible_hostname }}" ansible.builtin.wait_for: + host: "{{ ansible_hostname }}" port: 9200 timeout: 30 connect_timeout: 1 diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 092ac9c7a..df1610dbd 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -3,7 +3,7 @@ - name: Install elasticsearch become: true block: - # https://github.com/buluma/ansible-role-elasticsearch/issues/25 + # https://github.com/elastic/elasticsearch/issues/57018 - name: Create alternative tmp directory for elasticsearch ansible.builtin.file: path: "{{ elasticsearch_tmp_path }}" @@ -24,10 +24,13 @@ dest: "{{ elasticsearch_systemd_service_path }}/tmpdir.conf" mode: "0644" - - name: Enable elasticsearch repository - ansible.builtin.include_role: - name: buluma.elastic_repo - - name: Install elasticsearch ansible.builtin.include_role: - name: buluma.elasticsearch + name: geerlingguy.elasticsearch + vars: + elasticsearch_network_host: "{{ ansible_hostname }}" + elasticsearch_heap_size_min: 1g + elasticsearch_heap_size_max: 1g + elasticsearch_extra_options: | + xpack.security.enabled: false + discovery.type: single-node diff --git a/scripts/vagrant_provision.sh b/scripts/vagrant_provision.sh index 04281ca6f..004526902 100644 --- a/scripts/vagrant_provision.sh +++ b/scripts/vagrant_provision.sh @@ -3,4 +3,4 @@ curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python cd /vagrant python3 -m pipenv install --deploy python3 -m pipenv run ansible-galaxy install -r requirements.yml -python3 -m pipenv run ansible-playbook -i inventory_local.yml -e "autogen_unsecure_secrets=true" playbooks/acs.yml +python3 -m pipenv run ansible-playbook -i inventory_local.yml -e "autogen_unsecure_secrets=true" -e "known_urls=[http://localhost]" playbooks/acs.yml