Merge pull request #479 from mageops/dev-opensearch

Add support for opensearch 2

Author: Szpadel

group_vars/all.yml

@@ -561,7 +561,8 @@ aws_logs_slack_notifications: "{{ mageops_notifications_slack_enable }}"
 
 aws_logs_loggers_persistent_default:
   syslog: { enabled: yes }
-  elasticsearch-main: { enabled: "{{ mageops_elasticsearch_create }}" }
+  elasticsearch-main: { enabled: "{{  mageops_elasticsearch_opensearch_flavor == 'elasticsearch' }}" }
+  opensearch-main: { enabled: "{{  mageops_elasticsearch_opensearch_flavor == 'opensearch' }}" }
   redis-server: { enabled: "{{ redis_valkey_flavor == 'redis' }}" }
   redis-sessions-server: { enabled: "{{ mageops_redis_sessions_create }}" }
   valkey-server: { enabled: "{{ redis_valkey_flavor == 'valkey' }}" }
@@ -709,9 +710,12 @@ mageops_redis_sessions_create: yes
 mageops_redis_sessions_host:
 mageops_redis_sessions_port: 6380
 
-# Enable to provision Elasticsearch (or provide host)
-mageops_elasticsearch_create: yes
+mageops_elasticsearch_opensearch_flavor: elasticsearch
 # mageops_elasticsearch_host:
+mageops_elasticsearch_http_auth: false
+
+# mageops_opensearch_host:
+mageops_opensearch_http_auth: "{{ mageops_elasticsearch_http_auth }}"
 
 # Do not wait for Elasticsearch to be ready, required when aws opensearch serverless is used
 # because it required complex authentication that isn't worth implementing
@@ -720,6 +724,9 @@ mageops_skip_elasticsearch_check: no
 # Use https for connecting to Elasticsearch
 mageops_elasticsearch_https: no
 
+# Use https for connecting to Elasticsearch
+mageops_opensearch_https: "{{ mageops_elasticsearch_https }}"
+
 mageops_varnish_purge_logging: yes
 mageops_varnish_host: 127.0.0.1
 mageops_varnish_port: "{% if mageops_https_termination_enable %}8000{% else %}80{% endif %}"
@@ -952,6 +959,8 @@ magento_gid: "{{ mageops_app_gid }}"
 magento_umask: "{{ mageops_app_umask }}"
 
 magento_elasticsearch_index_alias: "{{ mageops_app_name }}"
+magento_opensearch_index_alias: "{{ mageops_app_name }}"
+
 
 mageops_install_magerun: yes
 magento_consumer_workers_enable: yes
@@ -1568,7 +1577,17 @@ elasticsearch_plugins:
     - analysis-icu
     - analysis-phonetic
 
+# ---------------------------------
+# --------  Opensearch  --------
+# ---------------------------------
+
+opensearch_cluster_name: "{{ mageops_app_id }}"
+opensearch_node_name: master
 
+opensearch_http_port: "{{ elasticsearch_http_port }}"
+opensearch_transport_tcp_port: "{{ elasticsearch_transport_tcp_port }}"
+opensearch_heap_size: "{{ elasticsearch_heap_size }}"
+opensearch_plugins: "{{ elasticsearch_plugins }}"
 
 
 # -------------------------
@@ -1785,8 +1804,8 @@ firewall_internal_services:
     - mysql
 
 firewall_internal_ports:
-    - "{{ elasticsearch_http_port }}/tcp"
-    - "{{ elasticsearch_transport_tcp_port }}/tcp"
+    - "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port ~ '/tcp', opensearch_http_port ~ '/tcp') }}"
+    - "{{ search_transport_tcp_port }}/tcp"
     - "{{ mageops_redis_port }}/tcp"
 firewall_public_services:
     - http

roles/cs.aws-logs/defaults/main.yml

@@ -87,6 +87,13 @@ aws_logs_loggers_default:
         retention: 7
         multiline: yes
         enabled: no
+    opensearch-main:
+        group_name: "/opensearch/node.log"
+        filename: "/var/log/opensearch/{{ opensearch_cluster_name }}.log"
+        date_format: "[%Y-%m-%dT%H:%M:%S"
+        retention: 7
+        multiline: yes
+        enabled: no
     redis-server:
         group_name: "/redis/server.log"
         filename: "/var/log/redis/redis-server.log"

roles/cs.aws-node-facts/tasks/main.yml

@@ -59,6 +59,12 @@
     mageops_elasticsearch_host: "{{ aws_nodes_info.elasticsearch.private_ip_address }}"
   when: not mageops_elasticsearch_host | default(false, true) and aws_nodes_info.elasticsearch
 
+- name: Set opensearch host if not predefined
+  set_fact:
+    mageops_opensearch_host: "{{ aws_nodes_info.elasticsearch.private_ip_address }}"
+  when: not mageops_opensearch_host | default(false, true) and aws_nodes_info.elasticsearch
+
+
 # TODO: Find RabbitMQ through tags, right now we don't have option to separate it anyway
 - name: Set RabbitMQ host if not predefined
   set_fact:

roles/cs.aws-node-persistent/tasks/main.yml

@@ -19,7 +19,7 @@
           aws_tags_default | combine(
             aws_tags_role_persistent,
             aws_tags_node_persistent,
-            mageops_elasticsearch_create | ternary(aws_tags_role_elasticsearch, {}),
+            (mageops_elasticsearch_opensearch_flavor in ['elasticsearch','opensearch']) | ternary(aws_tags_role_elasticsearch, {}),
             mageops_rabbitmq_create | ternary(aws_tags_role_rabbitmq_message_queue, {}),
             (redis_valkey_flavor in ['redis', 'valkey']) | ternary(aws_tags_role_redis_cache, {}),
             aws_persistent_node_name_tags

roles/cs.aws-security-group/defaults/main.yml

@@ -1,4 +1,3 @@
-
 aws_security_group_rds_create: no
 aws_security_group_redis_create: no
 aws_security_group_elasticsearch_create: no

roles/cs.aws-security-group/tasks/main.yml

@@ -154,25 +154,25 @@
   register: aws_security_group_redis
   when: aws_security_group_redis_create
 
-- name: Create security group for Elastic
+- name: Create security group for Search
   ec2_group:
     name: "{{ aws_security_group_elasticsearch_name }}"
-    description: "{{ mageops_app_name }}  Elasticsearch security group"
+    description: "{{ mageops_app_name }} search security group"
     region: "{{ aws_region }}"
     rules:
       - proto: tcp
-        ports: "{{ elasticsearch_http_port }}"
+        ports: "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port , opensearch_http_port) }}"
         group_name: "{{ aws_security_group_app_name }}"
       - proto: tcp
-        ports: "{{ elasticsearch_http_port }}"
+        ports: "{{ (mageops_elasticsearch_opensearch_flavor == 'elasticsearch') | ternary(elasticsearch_http_port, opensearch_http_port) }}"
         cidr_ip: "{{ mageops_trusted_cidr_blocks }}"
     vpc_id: "{{ aws_vpc_id }}"
     tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}"
   vars:
     ec2_sg_tags:
       Name: "{{ aws_security_group_elasticsearch_name }}"
-  register: aws_security_group_elastic
-  when: aws_security_group_elasticsearch_create
+  register: aws_security_group_search
+  when: aws_security_group_elastic_create
 
 - name: Create security group for EFS
   ec2_group:

roles/cs.elasticsearch/tasks/main.yml

@@ -52,6 +52,8 @@
   uri:
     url: "http://{{ elasticsearch_network_host }}:{{ elasticsearch_http_port }}"
     return_content: yes
+    url_password: "{{ mageops_elasticsearch_username | default(omit) }}"
+    url_username: "{{ mageops_elasticsearch_password | default(omit) }}"
   register: elasticsearch_get_running_version
   failed_when: false
   changed_when: false
@@ -147,6 +149,13 @@
   loop: "{{ elasticsearch_plugins }}"
   notify: Restart elasticsearch
 
+- name: Ensure opensearch is stopped and disabled
+  service:
+    name: opensearch
+    state: stopped
+    enabled: no
+  ignore_errors: true
+
 - name: Ensure elasticsearch is started and enabled
   service:
     name: elasticsearch
@@ -165,7 +174,7 @@
 
 - name: Fix deprecation indexes settings
   shell: >-
-    curl -XPUT -k "http://localhost:9200/_component_template/.deprecation-indexing-settings" -H 'Content-Type: application/json' -d '{"template":{"settings":{"index":{"lifecycle":{"name":".deprecation-indexing-ilm-policy"},"codec":"best_compression","hidden":"true","number_of_replicas":"0","auto_expand_replicas":"0-1","query":{"default_field":["message"]}}}},"version":1,"_meta":{"managed":true,"description":"default settings for Stack deprecation logs index template installed by x-pack"}}}'
+    curl -XPUT -k "http://localhost:{{elasticsearch_http_port}}/_component_template/.deprecation-indexing-settings" -H 'Content-Type: application/json' -d '{"template":{"settings":{"index":{"lifecycle":{"name":".deprecation-indexing-ilm-policy"},"codec":"best_compression","hidden":"true","number_of_replicas":"0","auto_expand_replicas":"0-1","query":{"default_field":["message"]}}}},"version":1,"_meta":{"managed":true,"description":"default settings for Stack deprecation logs index template installed by x-pack"}}}'
   when: elasticsearch_version_number is version('7.0.0', '>=')
 
 - name: Install elasticsearch-hprof

roles/cs.elasticsearch/templates/elasticsearch.yml

@@ -12,8 +12,12 @@ discovery.type: single-node
 
 {% if elasticsearch_version_number is version('7.16.0', '>=') %}
 cluster.deprecation_indexing.enabled: false
+{% if mageops_elasticsearch_http_auth %}
+xpack.security.enabled: true
+{% else %}
 xpack.security.enabled: false
 {% endif %}
+{% endif %}
 
 path.data: /var/lib/elasticsearch
 path.logs: /var/log/elasticsearch

roles/cs.magento-configure/defaults/main/app-etc.yml

@@ -103,12 +103,28 @@ magento_app_etc_config_elasticsuite:
         es_client:
           servers: "{{ mageops_elasticsearch_host }}:{{ elasticsearch_http_port }}"
           enable_https_mode: "{{ mageops_elasticsearch_https }}"
-          http_auth_user: ""
-          http_auth_pwd: ""
-          enable_http_auth: false
+          http_auth_user: "{{ mageops_elasticsearch_username | default('') }}"
+          http_auth_pwd: "{{ mageops_elasticsearch_password | default('') }}"
+          enable_http_auth: "{{ mageops_elasticsearch_http_auth | default(false) }}"
+          force_basic_auth: "{{ mageops_elasticsearch_http_auth | ternary(true, omit) }}"
         indices_settings:
           alias: "{{ magento_elasticsearch_index_alias }}"
 
+# Optional elasticsuite config with opensearch
+magento_app_etc_config_opensearch:
+  system:
+    default:
+      smile_elasticsuite_core_base_settings:
+        es_client:
+          servers: "{{ mageops_opensearch_host }}:{{ opensearch_http_port }}"
+          enable_https_mode: "{{ mageops_opensearch_https }}"
+          http_auth_user: "{{ mageops_opensearch_username | default('') }}"
+          http_auth_pwd: "{{ mageops_opensearch_password | default('') }}"
+          enable_http_auth: "{{ mageops_opensearch_http_auth | default(false) }}"
+          force_basic_auth: "{{ mageops_opensearch_http_auth | ternary(true, omit) }}"
+        indices_settings:
+          alias: "{{ magento_opensearch_index_alias }}"
+
 # Optional redis cache config
 magento_app_etc_config_cache_default_redis:
   cache:

roles/cs.magento-configure/tasks/037-wait-for-services.yml

@@ -1,14 +1,24 @@
 # TODO: This needs to be done on the final app node before setup:upgrade / cache:flush
 
 - name: Wait for Elasticsearch
-  shell: curl -sfk 'http://{{ mageops_elasticsearch_host }}:{{ elasticsearch_http_port }}'
+  shell: curl -u {{ mageops_elasticsearch_username | default('elastic')}}:{{mageops_elasticsearch_password | default('changeme')}} -sfk 'http://{{ mageops_elasticsearch_host }}:{{ elasticsearch_http_port }}'
   args:
     warn: no
   retries: 15
   delay: 5
   register: result
   until: result.rc == 0
-  when: mageops_elasticsearch_host | default(false, true) and not mageops_skip_elasticsearch_check
+  when: mageops_elasticsearch_host | default(false, true) and not mageops_skip_elasticsearch_check and mageops_elasticsearch_opensearch_flavor == "elasticsearch"
+
+- name: Wait for Opensearch
+  shell: curl -u {{ mageops_opensearch_username | default('elastic')}}:{{mageops_elasticsearch_password | default('changeme')}} -sfk 'http://{{ mageops_opensearch_host }}:{{ opensearch_http_port }}'
+  args:
+    warn: no
+  retries: 15
+  delay: 5
+  register: result
+  until: result.rc == 0
+  when: mageops_openearch_host | default(false, true) and mageops_elasticsearch_opensearch_flavor == "opensearch"
 
 - name: Wait for Redis cache host
   shell: echo -ne "PING\r\n" | nc {{ mageops_redis_host }} {{ mageops_redis_port }}