From 9a14e97c3f1c933cfee6f8fc0cb760d8e10252c1 Mon Sep 17 00:00:00 2001 From: Logan L Date: Thu, 7 May 2020 17:52:19 -0600 Subject: [PATCH 1/4] Add elasticsearch/export_index.sh --- docker-compose.yml | 11 +++++- elasticsearch/export_index.sh | 37 +++++++++++++++++++ .../BeaKer/elasticsearch/export_index.sh | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100755 elasticsearch/export_index.sh create mode 120000 installer/stage/BeaKer/elasticsearch/export_index.sh diff --git a/docker-compose.yml b/docker-compose.yml index 20cc2a7..a74ada5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,4 +25,13 @@ services: volumes: - ${BEAKER_CONFIG_DIR:-/etc/BeaKer/}certificates:/usr/share/kibana/config/certificates depends_on: - - elasticsearch \ No newline at end of file + - elasticsearch + + es-dump: + image: taskrabbit/elasticsearch-dump:v6.28.0 + restart: "no" + environment: + - NODE_TLS_REJECT_UNAUTHORIZED=0 + depends_on: + - elasticsearch + entrypoint: ["/bin/true"] diff --git a/elasticsearch/export_index.sh b/elasticsearch/export_index.sh new file mode 100755 index 0000000..b1728f3 --- /dev/null +++ b/elasticsearch/export_index.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Grab input arguments +if [ "$#" -ne 2 ]; then + echo "Usage: ./export_index.sh index_to_save output/directory" + exit 1 +fi + +index_name="$1" +output_dir=`realpath "$2"` + +# Change dir to script parent dir after calling realpath on the output directory +pushd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null + +# Use main function for local vars to hold sensitive info +main() { + + mkdir -p "$output_dir/$index_name" + + + local username="" + IFS="" read -es -p "Elasticsearch Username: " username + echo "" + local password="" + IFS="" read -es -p "Elasticsearch Password: " password + echo "" + + ./beaker run -v "$output_dir/$index_name:/exports:rw" --entrypoint multielasticdump es-dump --input="https://$username:$password@elasticsearch:9200" --output="/exports" --includeType="data,mapping" --match="^$index_name\$" + + tar -C "$output_dir" -czf "$output_dir/$index_name.tar.gz" "$index_name" + rm -rf "$output_dir/$index_name" +} + +main + +# Change back to original directory +popd > /dev/null diff --git a/installer/stage/BeaKer/elasticsearch/export_index.sh b/installer/stage/BeaKer/elasticsearch/export_index.sh new file mode 120000 index 0000000..d654aae --- /dev/null +++ b/installer/stage/BeaKer/elasticsearch/export_index.sh @@ -0,0 +1 @@ +../../../../elasticsearch/export_index.sh \ No newline at end of file From 5ca8c638850f65f97ce06d3a1109c8b877c83495 Mon Sep 17 00:00:00 2001 From: Logan L Date: Tue, 12 May 2020 16:58:48 -0600 Subject: [PATCH 2/4] Finished basic import script and tested it --- import_index.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 import_index.sh diff --git a/import_index.sh b/import_index.sh new file mode 100755 index 0000000..5970c29 --- /dev/null +++ b/import_index.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Grab input arguments +if [ "$#" -ne 1 ]; then + echo "Usage: ./import_index.sh path/to/archive.tar" + exit 1 +fi + +input_tar=`realpath "$1"` + +# Change dir to script parent dir after calling realpath on the output directory +pushd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null + +# Use main function for local vars to hold sensitive info +main() { + local input_tar_dir=`dirname "$input_tar"` + local index_name=`tar --exclude='*/*' -tf "$input_tar"` + + tar -xf "$input_tar" -C "$input_tar_dir" + local index_dir="$input_tar_dir/$index_name" + + local username="" + IFS="" read -es -p "Elasticsearch Username: " username + echo "" + local password="" + IFS="" read -es -p "Elasticsearch Password: " password + echo "" + ./beaker run -v "$index_dir:/$index_name" --entrypoint multielasticdump es-dump --direction=load --input="/$index_name" --output="https://$username:$password@elasticsearch:9200" + + rm -rf "$index_dir" +} + +main + +# Change back to original directory +popd > /dev/null From 491ecf1cc11def813b1e0dba4d74c36b2ef161e8 Mon Sep 17 00:00:00 2001 From: Logan L Date: Tue, 12 May 2020 17:06:25 -0600 Subject: [PATCH 3/4] Move import index script into its proper place --- import_index.sh | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100755 import_index.sh diff --git a/import_index.sh b/import_index.sh deleted file mode 100755 index 5970c29..0000000 --- a/import_index.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Grab input arguments -if [ "$#" -ne 1 ]; then - echo "Usage: ./import_index.sh path/to/archive.tar" - exit 1 -fi - -input_tar=`realpath "$1"` - -# Change dir to script parent dir after calling realpath on the output directory -pushd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null - -# Use main function for local vars to hold sensitive info -main() { - local input_tar_dir=`dirname "$input_tar"` - local index_name=`tar --exclude='*/*' -tf "$input_tar"` - - tar -xf "$input_tar" -C "$input_tar_dir" - local index_dir="$input_tar_dir/$index_name" - - local username="" - IFS="" read -es -p "Elasticsearch Username: " username - echo "" - local password="" - IFS="" read -es -p "Elasticsearch Password: " password - echo "" - ./beaker run -v "$index_dir:/$index_name" --entrypoint multielasticdump es-dump --direction=load --input="/$index_name" --output="https://$username:$password@elasticsearch:9200" - - rm -rf "$index_dir" -} - -main - -# Change back to original directory -popd > /dev/null From a4a31467fdcd271ab59c2f2725f18d7dbc2b4a78 Mon Sep 17 00:00:00 2001 From: Logan L Date: Tue, 12 May 2020 17:06:38 -0600 Subject: [PATCH 4/4] Move import index script into its proper place --- elasticsearch/import_index.sh | 36 +++++++++++++++++++ .../BeaKer/elasticsearch/import_index.sh | 1 + 2 files changed, 37 insertions(+) create mode 100755 elasticsearch/import_index.sh create mode 120000 installer/stage/BeaKer/elasticsearch/import_index.sh diff --git a/elasticsearch/import_index.sh b/elasticsearch/import_index.sh new file mode 100755 index 0000000..5970c29 --- /dev/null +++ b/elasticsearch/import_index.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Grab input arguments +if [ "$#" -ne 1 ]; then + echo "Usage: ./import_index.sh path/to/archive.tar" + exit 1 +fi + +input_tar=`realpath "$1"` + +# Change dir to script parent dir after calling realpath on the output directory +pushd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null + +# Use main function for local vars to hold sensitive info +main() { + local input_tar_dir=`dirname "$input_tar"` + local index_name=`tar --exclude='*/*' -tf "$input_tar"` + + tar -xf "$input_tar" -C "$input_tar_dir" + local index_dir="$input_tar_dir/$index_name" + + local username="" + IFS="" read -es -p "Elasticsearch Username: " username + echo "" + local password="" + IFS="" read -es -p "Elasticsearch Password: " password + echo "" + ./beaker run -v "$index_dir:/$index_name" --entrypoint multielasticdump es-dump --direction=load --input="/$index_name" --output="https://$username:$password@elasticsearch:9200" + + rm -rf "$index_dir" +} + +main + +# Change back to original directory +popd > /dev/null diff --git a/installer/stage/BeaKer/elasticsearch/import_index.sh b/installer/stage/BeaKer/elasticsearch/import_index.sh new file mode 120000 index 0000000..27bd177 --- /dev/null +++ b/installer/stage/BeaKer/elasticsearch/import_index.sh @@ -0,0 +1 @@ +../../../../elasticsearch/import_index.sh \ No newline at end of file