OBS-486: remove legacy elasticsearch (#6906)

* OBS-486: remove legacy elasticsearch

* address review

Author: relud

.devcontainer/Dockerfile

@@ -31,17 +31,6 @@ RUN /tmp/set_up_stackwalker.sh && \
 COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
 RUN cd /webapp-frontend-deps/ && npm install
 
-# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the
-# new location (aka shaded), so that they can be installed at the same time as newer versions
-COPY --chown=app:app legacy-es-requirements.txt /app/
-RUN pip install -U 'pip==24.0' && \
-    pip install --no-cache-dir --no-deps -r legacy-es-requirements.txt && \
-    cd /usr/local/lib/python3.11/site-packages/ && \
-    mv elasticsearch elasticsearch_1_9_0 && \
-    mv elasticsearch_dsl elasticsearch_dsl_0_0_11 && \
-    cd elasticsearch_dsl_0_0_11/ && \
-    sed 's/^from elasticsearch/from elasticsearch_1_9_0/' -i connections.py search.py serializer.py
-
 # Install Python dependencies
 COPY requirements.txt /app/
 RUN pip install --no-cache-dir --no-deps -r requirements.txt && \

bin/es_cli.py

@@ -13,8 +13,7 @@
 import json
 
 import click
-from elasticsearch_dsl_0_0_11 import Search
-from elasticsearch_1_9_0.client import IndicesClient
+from elasticsearch_dsl import Search
 
 from socorro import settings
 from socorro.external.es.base import generate_list_of_indexes
@@ -107,8 +106,7 @@ def cmd_print_mapping(ctx, index):
     crashstorage = get_crashstorage()
     doctype = crashstorage.get_doctype()
     with crashstorage.client() as conn:
-        indices_client = IndicesClient(conn)
-        resp = indices_client.get_mapping(index=index)
+        resp = conn.indices.get_mapping(index=index)
         mapping = resp[index]["mappings"][doctype]["properties"]
         click.echo(json.dumps(mapping, indent=2, sort_keys=True))
 

bin/legacy_es_cli.py

@@ -73,11 +73,6 @@ def cmd_permadelete(ctx, crashidsfile):
 
     crashstorage = build_instance_from_settings(settings.STORAGE)
     es_crashstorage = build_instance_from_settings(settings.ES_STORAGE)
-    legacy_es_crashstorage = None
-    if settings.ELASTICSEARCH_MODE == "PREFER_NEW":
-        legacy_es_crashstorage = build_instance_from_settings(
-            settings.LEGACY_ES_STORAGE
-        )
 
     crashids = crashid_generator(crashidsfile)
 
@@ -87,29 +82,16 @@ def cmd_permadelete(ctx, crashidsfile):
 
         data = crashstorage.catalog_crash(crash_id=crashid)
         data.extend(es_crashstorage.catalog_crash(crash_id=crashid))
-        if legacy_es_crashstorage:
-            data.extend(
-                f"legacy_{v}"
-                for v in legacy_es_crashstorage.catalog_crash(crash_id=crashid)
-            )
         click.echo(f">>> before: {data}")
 
         # Delete from storage
         click.echo(">>> Deleting from storage ...")
         crashstorage.delete_crash(crashid)
         click.echo(">>> Deleting from Elasticsearch ...")
         es_crashstorage.delete_crash(crashid)
-        if legacy_es_crashstorage:
-            click.echo(">>> Deleting from legacy Elasticsearch ...")
-            legacy_es_crashstorage.delete_crash(crashid)
 
         data = crashstorage.catalog_crash(crash_id=crashid)
         data.extend(es_crashstorage.catalog_crash(crash_id=crashid))
-        if legacy_es_crashstorage:
-            data.extend(
-                f"legacy_{v}"
-                for v in legacy_es_crashstorage.catalog_crash(crash_id=crashid)
-            )
         click.echo(f">>> after: {data}")
 
         end_time = datetime.datetime.now()

bin/permadelete_crash_data.py

@@ -27,13 +27,8 @@ gcs-cli delete "${TELEMETRY_GCS_BUCKET}"
 gcs-cli create "${TELEMETRY_GCS_BUCKET}"
 
 # Delete and create Elasticsearch indices
-/app/socorro-cmd legacy_es delete
-/app/socorro-cmd legacy_es create
-ELASTICSEARCH_MODE="${ELASTICSEARCH_MODE:-LEGACY_ONLY}"
-if [ "${ELASTICSEARCH_MODE^^}" == "PREFER_NEW" ]; then
-    /app/socorro-cmd es delete
-    /app/socorro-cmd es create
-fi
+/app/socorro-cmd es delete
+/app/socorro-cmd es create
 
 # Delete and create Pub/Sub queues
 pubsub-cli delete-topic "$PUBSUB_PROJECT_ID" "$PUBSUB_STANDARD_TOPIC_NAME"

bin/setup_services.sh

@@ -19,7 +19,6 @@ echo ">>> set up environment"
 DATABASE_URL="${DATABASE_URL:-}"
 SENTRY_DSN="${SENTRY_DSN:-}"
 ELASTICSEARCH_URL="${ELASTICSEARCH_URL:-}"
-LEGACY_ELASTICSEARCH_URL="${LEGACY_ELASTICSEARCH_URL:-}"
 
 export PYTHONPATH=/app/:$PYTHONPATH
 PYTEST="$(which pytest)"
@@ -47,14 +46,10 @@ popd
 echo ">>> run tests"
 
 # Run socorro tests
-# ignore tests because elasticsearch 8 tests cannot be run with LEGACY_ONLY
-ELASTICSEARCH_MODE=LEGACY_ONLY "${PYTEST}" --ignore=socorro/tests/external/es/
-# ignore tests because elasticsearch 8 supersearch is not implemented yet
-ELASTICSEARCH_MODE=PREFER_NEW "${PYTEST}"
+"${PYTEST}"
 
 # Collect static and then run pytest in the webapp
 pushd webapp
 ${PYTHON} manage.py collectstatic --noinput
-ELASTICSEARCH_MODE=LEGACY_ONLY "${PYTEST}"
-ELASTICSEARCH_MODE=PREFER_NEW "${PYTEST}"
+"${PYTEST}"
 popd

bin/test.sh

@@ -14,7 +14,3 @@ set -euo pipefail
 cp requirements.txt requirements.txt.orig
 pip-compile --quiet --generate-hashes --strip-extras
 diff requirements.txt requirements.txt.orig
-
-cp legacy-es-requirements.txt legacy-es-requirements.txt.orig
-pip-compile --quiet --generate-hashes --unsafe-package=python-dateutil --unsafe-package=six --unsafe-package=urllib3 legacy-es-requirements.in
-diff legacy-es-requirements.txt legacy-es-requirements.txt.orig

bin/verify_reqs.sh

@@ -13,7 +13,6 @@
 set -euo pipefail
 
 waitfor --verbose --conn-only "${DATABASE_URL}"
-waitfor --verbose "${LEGACY_ELASTICSEARCH_URL}"
 waitfor --verbose "http://${PUBSUB_EMULATOR_HOST}"
 waitfor --verbose "${STORAGE_EMULATOR_HOST}/storage/v1/b"
 waitfor --verbose --codes={200,404} "${SENTRY_DSN}"

bin/waitfor_services.sh

@@ -25,7 +25,6 @@ services:
       - gcs-emulator
       - pubsub
       - postgresql
-      - legacy-elasticsearch
       - elasticsearch
 
   # For development
@@ -42,7 +41,6 @@ services:
       - gcs-emulator
       - pubsub
       - postgresql
-      - legacy-elasticsearch
       - elasticsearch
 
   devcontainer:
@@ -63,7 +61,6 @@ services:
       - gcs-emulator
       - pubsub
       - postgresql
-      - legacy-elasticsearch
       - elasticsearch
     volumes:
       - .:/app
@@ -78,7 +75,6 @@ services:
       - statsd
       - gcs-emulator
       - pubsub
-      - legacy-elasticsearch
       - elasticsearch
       - symbolsserver
     command: ["processor"]
@@ -92,7 +88,6 @@ services:
       - fakesentry
       - statsd
       - postgresql
-      - legacy-elasticsearch
       - elasticsearch
     command: ["crontabber"]
 
@@ -107,7 +102,6 @@ services:
       - gcs-emulator
       - pubsub
       - postgresql
-      - legacy-elasticsearch
       - elasticsearch
       - memcached
       - oidcprovider
@@ -200,12 +194,6 @@ services:
       - "8125:8125/udp"
       - "8126:8126"
 
-  # https://hub.docker.com/_/elasticsearch/
-  legacy-elasticsearch:
-    image: mozilla/socorro_elasticsearch:1.4.5
-    ports:
-      - "9200:9200"
-
   # https://www.elastic.co/guide/en/elasticsearch/reference/8.16/docker.html
   elasticsearch:
     build:
@@ -217,7 +205,7 @@ services:
       - -Expack.security.enabled=false
       - -Ediscovery.type=single-node
     ports:
-      - "9201:9200"
+      - "9200:9200"
 
   # https://hub.docker.com/_/postgres/
   postgresql:

docker-compose.yml

@@ -30,13 +30,6 @@ RUN /tmp/set_up_stackwalker.sh && \
 COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
 RUN cd /webapp-frontend-deps/ && npm install
 
-# NOTE(relud): these dependencies are installed separately, relocated, and patched to reference the
-# new location (aka shaded), so that they can be installed at the same time as newer versions
-COPY --chown=app:app legacy-es-requirements.txt /app/
-COPY docker/set_up_legacy_es_requirements.sh /tmp/set_up_legacy_es_requirements.sh
-RUN /tmp/set_up_legacy_es_requirements.sh && \
-    rm /tmp/set_up_legacy_es_requirements.sh
-
 COPY --chown=app:app requirements.txt /app/
 RUN pip install --no-cache-dir --no-deps -r requirements.txt && \
     pip check --disable-pip-version-check

docker/Dockerfile

@@ -24,7 +24,6 @@ STATSD_HOST=statsd
 # -------------
 
 ELASTICSEARCH_URL=http://elasticsearch:9200
-LEGACY_ELASTICSEARCH_URL=http://legacy-elasticsearch:9200
 
 
 # processor