[RFS] Collection of Minor Changes (opensearch-project#717)

* Swapped RFS Docker Compose to use ES 7.17 Signed-off-by: Chris Helma <chelma+github@amazon.com> * Updated RFS to delete files when finished, reject large shards Signed-off-by: Chris Helma <chelma+github@amazon.com> * Updates per PR comments Signed-off-by: Chris Helma <chelma+github@amazon.com> * Small RFS bug fix Signed-off-by: Chris Helma <chelma+github@amazon.com> * More updates per PR comments Signed-off-by: Chris Helma <chelma+github@amazon.com> * More updates per PR discussion Signed-off-by: Chris Helma <chelma+github@amazon.com> --------- Signed-off-by: Chris Helma <chelma+github@amazon.com>
chelma · Jun 13, 2024 · 055e07a · 055e07a
1 parent 254d847
commit 055e07a
Show file tree

Hide file tree

Showing 20 changed files with 382 additions and 366 deletions.
diff --git a/RFS/build-preloaded-source-image.gradle b/RFS/build-preloaded-source-image.gradle
@@ -23,8 +23,8 @@ def createNetworkTask = task createNetwork(type: Exec) {
     }
 }
 task createInitialElasticsearchContainer(type: DockerCreateContainer) {
-    dependsOn createNetwork, buildDockerImage_emptyElasticsearchSource
-    targetImageId 'migrations/empty_elasticsearch_source:latest'
+    dependsOn createNetwork, buildDockerImage_emptyElasticsearchSource_7_17
+    targetImageId 'migrations/emptyElasticsearchSource_7_17:latest'
     containerName = "elasticsearch-${uniqueId}"
     hostConfig.network = myNetworkName
     hostConfig.dns = ['elasticsearch']

diff --git a/RFS/build.gradle b/RFS/build.gradle
@@ -108,11 +108,6 @@ jacocoTestReport {
     }
 }
 
-task demoPrintOutSnapshot (type: JavaExec) {
-    classpath = sourceSets.main.runtimeClasspath
-    mainClass = 'com.rfs.DemoPrintOutSnapshot'
-}
-
 task copyDockerRuntimeJars (type: Sync) {
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
     description = 'Copy runtime JARs and app jar to docker build directory'
@@ -132,9 +127,12 @@ DockerServiceProps[] dockerServices = [
                                 dockerImageName:"reindex_from_snapshot",
                                 inputDir:"./docker",
                                 taskDependencies:["copyDockerRuntimeJars"]]),
-        new DockerServiceProps([projectName:"emptyElasticsearchSource",
-                                dockerImageName:"empty_elasticsearch_source",
+        new DockerServiceProps([projectName:"emptyElasticsearchSource_7_10",
+                                dockerImageName:"empty_elasticsearch_source_7_10",
                                 inputDir:"./docker/TestSource_ES_7_10"]),
+        new DockerServiceProps([projectName:"emptyElasticsearchSource_7_17",
+                                dockerImageName:"empty_elasticsearch_source_7_17",
+                                inputDir:"./docker/TestSource_ES_7_17"]),
         new DockerServiceProps([projectName:"trafficGenerator",
                                 dockerImageName:"osb_traffic_generator",
                                 inputDir:"./docker/TrafficGenerator",

diff --git a/RFS/docker/TestSource_ES_7_17/Dockerfile b/RFS/docker/TestSource_ES_7_17/Dockerfile
@@ -0,0 +1,22 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.17.21 AS base
+
+# Configure Elastic
+ENV ELASTIC_SEARCH_CONFIG_FILE=/usr/share/elasticsearch/config/elasticsearch.yml
+# Prevents ES from complaining about nodes count
+RUN echo "discovery.type: single-node" >> $ELASTIC_SEARCH_CONFIG_FILE
+ENV PATH=${PATH}:/usr/share/elasticsearch/jdk/bin/
+
+# Install the S3 Repo Plugin
+RUN echo y | /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-s3
+
+# Install the AWS CLI for testing purposes
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install
+
+RUN mkdir /snapshots && chown elasticsearch /snapshots
+
+# Install our custom entrypoint script
+COPY ./container-start.sh /usr/share/elasticsearch/container-start.sh
+
+CMD /usr/share/elasticsearch/container-start.sh
diff --git a/RFS/docker/TestSource_ES_7_17/container-start.sh b/RFS/docker/TestSource_ES_7_17/container-start.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Setting AWS Creds from ENV Variables"
+bin/elasticsearch-keystore create
+echo $AWS_ACCESS_KEY_ID | bin/elasticsearch-keystore add s3.client.default.access_key --stdin
+echo $AWS_SECRET_ACCESS_KEY | bin/elasticsearch-keystore add s3.client.default.secret_key --stdin
+
+if [ -n "$AWS_SESSION_TOKEN" ]; then
+    echo $AWS_SESSION_TOKEN | bin/elasticsearch-keystore add s3.client.default.session_token --stdin
+fi
+
+echo "Starting Elasticsearch"
+/usr/local/bin/docker-entrypoint.sh eswrapper
diff --git a/RFS/src/main/java/com/rfs/DemoPrintOutSnapshot.java b/RFS/src/main/java/com/rfs/DemoPrintOutSnapshot.java
diff --git a/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java b/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java
@@ -329,7 +329,8 @@ public static void main(String[] args) throws InterruptedException {
                 } else {
                     bufferSize = ElasticsearchConstants_ES_7_10.BUFFER_SIZE_IN_BYTES;
                 }
-                SnapshotShardUnpacker unpacker = new SnapshotShardUnpacker(repo, luceneDirPath, bufferSize);
+                DefaultSourceRepoAccessor repoAccessor = new DefaultSourceRepoAccessor(repo);
+                SnapshotShardUnpacker.Factory unpackerFactory = new SnapshotShardUnpacker.Factory(repoAccessor,luceneDirPath, bufferSize);
 
                 for (IndexMetadata.Data indexMetadata : indexMetadatas) {
                     logger.info("Processing index: " + indexMetadata.getName());
@@ -345,7 +346,9 @@ public static void main(String[] args) throws InterruptedException {
                         }
 
                         // Unpack the shard
-                        unpacker.unpack(shardMetadata);
+                        try (SnapshotShardUnpacker unpacker = unpackerFactory.create(shardMetadata)) {
+                            unpacker.unpack();
+                        }        
                     }
                 }