Upgrade ES to 8.x (#10771)

* Upgrade ES to 8.x We need to wait for django-es/django-elasticsearch-dsl#458 or #10730. There are no breaking changes in the API, so this can be deployed without any downtime. The only change is that it now has authentication enabled by default, so our docker image now uses a password. How to deploy: - Deploy web extra - The new packages will continue working with the old ES, but we can make sure by searching from web-extra. - Finish the deploy - Upgrade the ES cluster from cloud.elastic.co * Forgot about this * Fix tests * Upgrade dependencies
readthedocs · Oct 9, 2023 · e32e707 · e32e707
1 parent 3499743
commit e32e707
Show file tree

Hide file tree

Showing 9 changed files with 59 additions and 46 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,11 +11,14 @@ jobs:
         environment:
           TOX_POSARGS: ''
           PYTEST_COVERAGE: --cov-report=xml --cov-config .coveragerc --cov=. --cov-append
-      - image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.13'
+      - image: 'docker.elastic.co/elasticsearch/elasticsearch:8.10.2'
         name: search
         environment:
           discovery.type: single-node
           ES_JAVA_OPTS: -Xms750m -Xmx750m
+          ELASTIC_PASSWORD: password
+          # Disabled SSL for testing.
+          xpack.security.transport.ssl.enabled: 'false'
     steps:
       - checkout
       - run: git submodule sync

diff --git a/readthedocs/search/tests/test_faceted_search.py b/readthedocs/search/tests/test_faceted_search.py
@@ -26,11 +26,11 @@ def test_search_exact_match(self, client, project, case):
         assert len(results) == 2
         assert results[0]["project"] == "kuma"
         assert results[0]["path"] == "testdocumentation"
-        assert results[0]["version"] == "stable"
+        assert results[0]["version"] == "latest"
 
         assert results[1]["project"] == "kuma"
         assert results[1]["path"] == "testdocumentation"
-        assert results[1]["version"] == "latest"
+        assert results[1]["version"] == "stable"
 
     def test_search_combined_result(self, client, project):
         """Check search result are combined of both `AND` and `OR` operator

diff --git a/readthedocs/search/tests/test_views.py b/readthedocs/search/tests/test_views.py
@@ -260,8 +260,8 @@ def test_file_search_exact_match(self, client, project):
         # There are two results,
         # one from each version of the "kuma" project.
         assert len(results) == 2
-        assert results[0]["version"] == {"slug": "stable"}
-        assert results[1]["version"] == {"slug": "latest"}
+        assert results[0]["version"] == {"slug": "latest"}
+        assert results[1]["version"] == {"slug": "stable"}
         for result in results:
             assert result["project"] == {"alias": None, "slug": "kuma"}
             assert result["domain"] == "http://readthedocs.org"

diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
@@ -795,10 +795,9 @@ def DOCKER_LIMITS(self):
     RTD_ORG_TRIAL_PERIOD_DAYS = 30
 
     # Elasticsearch settings.
-    ES_HOSTS = ['search:9200']
     ELASTICSEARCH_DSL = {
         'default': {
-            'hosts': 'search:9200'
+            'hosts': 'http://elastic:password@search:9200',
         },
     }
     # Chunk size for elasticsearch reindex celery tasks

diff --git a/requirements/deploy.txt b/requirements/deploy.txt
@@ -40,7 +40,7 @@ celery==5.2.7
 certifi==2023.7.22
     # via
     #   -r requirements/pip.txt
-    #   elasticsearch
+    #   elastic-transport
     #   requests
     #   sentry-sdk
 cffi==1.16.0
@@ -135,7 +135,7 @@ django-csp==3.7
     # via -r requirements/pip.txt
 django-debug-toolbar==4.2.0
     # via -r requirements/pip.txt
-django-elasticsearch-dsl==7.4
+django-elasticsearch-dsl==8.0
     # via -r requirements/pip.txt
 django-extensions==3.2.3
     # via -r requirements/pip.txt
@@ -155,7 +155,7 @@ django-polymorphic==3.1.0
     # via -r requirements/pip.txt
 django-simple-history==3.0.0
     # via -r requirements/pip.txt
-django-storages[boto3]==1.14.1
+django-storages[boto3]==1.14.2
     # via -r requirements/pip.txt
 django-structlog==2.2.0
     # via -r requirements/pip.txt
@@ -185,11 +185,15 @@ drf-extensions==0.7.1
     # via -r requirements/pip.txt
 drf-flex-fields==1.0.2
     # via -r requirements/pip.txt
-elasticsearch==7.17.9
+elastic-transport==8.4.1
+    # via
+    #   -r requirements/pip.txt
+    #   elasticsearch
+elasticsearch==8.10.0
     # via
     #   -r requirements/pip.txt
     #   elasticsearch-dsl
-elasticsearch-dsl==7.4.1
+elasticsearch-dsl==8.9.0
     # via
     #   -r requirements/pip.txt
     #   django-elasticsearch-dsl
@@ -341,7 +345,6 @@ six==1.16.0
     #   asttokens
     #   django-annoying
     #   django-elasticsearch-dsl
-    #   elasticsearch-dsl
     #   python-dateutil
     #   unicode-slugify
 slumber==0.7.1
@@ -357,7 +360,7 @@ stripe==4.2.0
     # via
     #   -r requirements/pip.txt
     #   dj-stripe
-structlog==23.1.0
+structlog==23.2.0
     # via
     #   -r requirements/pip.txt
     #   django-structlog
@@ -395,7 +398,7 @@ urllib3==1.26.17
     #   -r requirements/pip.txt
     #   botocore
     #   docker
-    #   elasticsearch
+    #   elastic-transport
     #   requests
     #   sentry-sdk
 user-agents==2.2.0
@@ -412,7 +415,7 @@ wcwidth==0.2.8
     # via
     #   -r requirements/pip.txt
     #   prompt-toolkit
-websocket-client==1.6.3
+websocket-client==1.6.4
     # via
     #   -r requirements/pip.txt
     #   docker
diff --git a/requirements/docker.txt b/requirements/docker.txt
@@ -44,7 +44,7 @@ celery==5.2.7
 certifi==2023.7.22
     # via
     #   -r requirements/pip.txt
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 cffi==1.16.0
     # via
@@ -146,7 +146,7 @@ django-csp==3.7
     # via -r requirements/pip.txt
 django-debug-toolbar==4.2.0
     # via -r requirements/pip.txt
-django-elasticsearch-dsl==7.4
+django-elasticsearch-dsl==8.0
     # via -r requirements/pip.txt
 django-extensions==3.2.3
     # via -r requirements/pip.txt
@@ -166,7 +166,7 @@ django-polymorphic==3.1.0
     # via -r requirements/pip.txt
 django-simple-history==3.0.0
     # via -r requirements/pip.txt
-django-storages[boto3]==1.14.1
+django-storages[boto3]==1.14.2
     # via -r requirements/pip.txt
 django-structlog==2.2.0
     # via -r requirements/pip.txt
@@ -196,11 +196,15 @@ drf-extensions==0.7.1
     # via -r requirements/pip.txt
 drf-flex-fields==1.0.2
     # via -r requirements/pip.txt
-elasticsearch==7.17.9
+elastic-transport==8.4.1
+    # via
+    #   -r requirements/pip.txt
+    #   elasticsearch
+elasticsearch==8.10.0
     # via
     #   -r requirements/pip.txt
     #   elasticsearch-dsl
-elasticsearch-dsl==7.4.1
+elasticsearch-dsl==8.9.0
     # via
     #   -r requirements/pip.txt
     #   django-elasticsearch-dsl
@@ -374,7 +378,6 @@ six==1.16.0
     #   asttokens
     #   django-annoying
     #   django-elasticsearch-dsl
-    #   elasticsearch-dsl
     #   python-dateutil
     #   unicode-slugify
 slumber==0.7.1
@@ -390,7 +393,7 @@ stripe==4.2.0
     # via
     #   -r requirements/pip.txt
     #   dj-stripe
-structlog==23.1.0
+structlog==23.2.0
     # via
     #   -r requirements/pip.txt
     #   django-structlog
@@ -430,7 +433,7 @@ urllib3==1.26.17
     #   -r requirements/pip.txt
     #   botocore
     #   docker
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 user-agents==2.2.0
     # via -r requirements/pip.txt
@@ -448,7 +451,7 @@ wcwidth==0.2.8
     # via
     #   -r requirements/pip.txt
     #   prompt-toolkit
-websocket-client==1.6.3
+websocket-client==1.6.4
     # via
     #   -r requirements/pip.txt
     #   docker

diff --git a/requirements/pip.in b/requirements/pip.in
@@ -87,9 +87,10 @@ django-allauth==0.51.0
 requests-oauthlib
 
 # Search
-elasticsearch<8.0
-elasticsearch-dsl<8.0
-django-elasticsearch-dsl<8.0
+elasticsearch~=8.0
+elasticsearch-dsl~=8.0
+django-elasticsearch-dsl~=8.0
+
 selectolax
 
 # embed

diff --git a/requirements/pip.txt b/requirements/pip.txt
@@ -24,7 +24,7 @@ celery==5.2.7
     #   django-celery-beat
 certifi==2023.7.22
     # via
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 cffi==1.16.0
     # via cryptography
@@ -95,7 +95,7 @@ django-csp==3.7
     # via -r requirements/pip.in
 django-debug-toolbar==4.2.0
     # via -r requirements/pip.in
-django-elasticsearch-dsl==7.4
+django-elasticsearch-dsl==8.0
     # via -r requirements/pip.in
 django-extensions==3.2.3
     # via -r requirements/pip.in
@@ -113,7 +113,7 @@ django-polymorphic==3.1.0
     # via -r requirements/pip.in
 django-simple-history==3.0.0
     # via -r requirements/pip.in
-django-storages[boto3]==1.14.1
+django-storages[boto3]==1.14.2
     # via -r requirements/pip.in
 django-structlog==2.2.0
     # via -r requirements/pip.in
@@ -141,11 +141,13 @@ drf-extensions==0.7.1
     # via -r requirements/pip.in
 drf-flex-fields==1.0.2
     # via -r requirements/pip.in
-elasticsearch==7.17.9
+elastic-transport==8.4.1
+    # via elasticsearch
+elasticsearch==8.10.0
     # via
     #   -r requirements/pip.in
     #   elasticsearch-dsl
-elasticsearch-dsl==7.4.1
+elasticsearch-dsl==8.9.0
     # via
     #   -r requirements/pip.in
     #   django-elasticsearch-dsl
@@ -241,7 +243,6 @@ six==1.16.0
     # via
     #   django-annoying
     #   django-elasticsearch-dsl
-    #   elasticsearch-dsl
     #   python-dateutil
     #   unicode-slugify
 slumber==0.7.1
@@ -254,7 +255,7 @@ stripe==4.2.0
     # via
     #   -r requirements/pip.in
     #   dj-stripe
-structlog==23.1.0
+structlog==23.2.0
     # via
     #   -r requirements/pip.in
     #   django-structlog
@@ -276,7 +277,7 @@ urllib3==1.26.17
     # via
     #   botocore
     #   docker
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 user-agents==2.2.0
     # via -r requirements/pip.in
@@ -289,7 +290,7 @@ virtualenv==20.24.5
     # via -r requirements/pip.in
 wcwidth==0.2.8
     # via prompt-toolkit
-websocket-client==1.6.3
+websocket-client==1.6.4
     # via docker
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/requirements/testing.txt b/requirements/testing.txt
@@ -40,7 +40,7 @@ celery==5.2.7
 certifi==2023.7.22
     # via
     #   -r requirements/pip.txt
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 cffi==1.16.0
     # via
@@ -136,7 +136,7 @@ django-debug-toolbar==4.2.0
     # via -r requirements/pip.txt
 django-dynamic-fixture==4.0.1
     # via -r requirements/testing.in
-django-elasticsearch-dsl==7.4
+django-elasticsearch-dsl==8.0
     # via -r requirements/pip.txt
 django-extensions==3.2.3
     # via -r requirements/pip.txt
@@ -156,7 +156,7 @@ django-polymorphic==3.1.0
     # via -r requirements/pip.txt
 django-simple-history==3.0.0
     # via -r requirements/pip.txt
-django-storages[boto3]==1.14.1
+django-storages[boto3]==1.14.2
     # via -r requirements/pip.txt
 django-structlog==2.2.0
     # via -r requirements/pip.txt
@@ -188,11 +188,15 @@ drf-extensions==0.7.1
     # via -r requirements/pip.txt
 drf-flex-fields==1.0.2
     # via -r requirements/pip.txt
-elasticsearch==7.17.9
+elastic-transport==8.4.1
+    # via
+    #   -r requirements/pip.txt
+    #   elasticsearch
+elasticsearch==8.10.0
     # via
     #   -r requirements/pip.txt
     #   elasticsearch-dsl
-elasticsearch-dsl==7.4.1
+elasticsearch-dsl==8.9.0
     # via
     #   -r requirements/pip.txt
     #   django-elasticsearch-dsl
@@ -355,7 +359,6 @@ six==1.16.0
     #   -r requirements/pip.txt
     #   django-annoying
     #   django-elasticsearch-dsl
-    #   elasticsearch-dsl
     #   python-dateutil
     #   requests-mock
     #   unicode-slugify
@@ -392,7 +395,7 @@ stripe==4.2.0
     # via
     #   -r requirements/pip.txt
     #   dj-stripe
-structlog==23.1.0
+structlog==23.2.0
     # via
     #   -r requirements/pip.txt
     #   django-structlog
@@ -425,7 +428,7 @@ urllib3==1.26.17
     #   -r requirements/pip.txt
     #   botocore
     #   docker
-    #   elasticsearch
+    #   elastic-transport
     #   requests
 user-agents==2.2.0
     # via -r requirements/pip.txt
@@ -441,7 +444,7 @@ wcwidth==0.2.8
     # via
     #   -r requirements/pip.txt
     #   prompt-toolkit
-websocket-client==1.6.3
+websocket-client==1.6.4
     # via
     #   -r requirements/pip.txt
     #   docker