From c7f0fb125ecaa273f8403bf4fe1ecb93c4a8aa95 Mon Sep 17 00:00:00 2001 From: Jose Quintana Date: Tue, 15 Mar 2022 13:04:06 +0100 Subject: [PATCH] chore: exporter and importer tools --- .gitignore | 13 +++ 8.0/Dockerfile | 13 +++ 8.0/env/mysql_exporter.env | 23 +++++ 8.0/env/mysql_importer.env | 23 +++++ 8.0/scripts/___mysqlexport.sh | 82 ++++++++++++++++++ 8.0/scripts/___mysqlimport.sh | 65 ++++++++++++++ 8.0/scripts/mysql_exporter | 17 ++++ 8.0/scripts/mysql_importer | 17 ++++ Makefile | 8 +- README.md | 155 ++++++++++++++++++++++++++++++++++ 10 files changed, 412 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 8.0/env/mysql_exporter.env create mode 100644 8.0/env/mysql_importer.env create mode 100755 8.0/scripts/___mysqlexport.sh create mode 100755 8.0/scripts/___mysqlimport.sh create mode 100755 8.0/scripts/mysql_exporter create mode 100755 8.0/scripts/mysql_importer diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dbce406 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.~ +**/*.tgz +**/*.gz +**/*.zip +**/*.gzip +**/*.log +**/*.tar +**/*.sql +**/.DS_Store +*.env + +!mysql_exporter.env +!mysql_importer.env diff --git a/8.0/Dockerfile b/8.0/Dockerfile index 77c0c5b..80e0492 100644 --- a/8.0/Dockerfile +++ b/8.0/Dockerfile @@ -99,7 +99,20 @@ RUN set -eux \ && rm -rf /var/lib/apt/lists/* \ && true +# Include the Enve tool +ARG ENVE_VERSION=1.4.0 + +RUN set -eux \ + && wget -O /tmp/enve.tar.gz \ + "https://github.com/joseluisq/enve/releases/download/v${ENVE_VERSION}/enve_v${ENVE_VERSION}_linux_amd64.tar.gz" \ + && tar xzvf /tmp/enve.tar.gz -C /usr/local/bin enve \ + && enve -v \ + && rm -rf /tmp/enve.tar.gz \ + && chmod +x /usr/local/bin/enve \ + && true + COPY 8.0/entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY 8.0/scripts/. /usr/local/bin/ VOLUME [ "/var/lib/mysql" ] diff --git a/8.0/env/mysql_exporter.env b/8.0/env/mysql_exporter.env new file mode 100644 index 0000000..641894c --- /dev/null +++ b/8.0/env/mysql_exporter.env @@ -0,0 +1,23 @@ +## MySQL 8 Export template settings +## -------------------------------- + +# Connection settings (optional) +DB_PROTOCOL=tcp +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DEFAULT_CHARACTER_SET=utf8 + +# GZip export file (optional) +DB_EXPORT_GZIP=false + +# SQL or Gzip export file (optional). +# If `DB_IMPORT_GZIP` is `true` then file name should be `database_name.sql.gz` +DB_EXPORT_FILE_PATH=database_name.sql + +# Database settings (required) +DB_NAME="" +DB_USERNAME="" +DB_PASSWORD="" + +# Additional arguments (optional) +DB_ARGS= diff --git a/8.0/env/mysql_importer.env b/8.0/env/mysql_importer.env new file mode 100644 index 0000000..da7685d --- /dev/null +++ b/8.0/env/mysql_importer.env @@ -0,0 +1,23 @@ +## MySQL 8 Import template settings +## -------------------------------- + +# Connection settings (optional) +DB_PROTOCOL=tcp +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DEFAULT_CHARACTER_SET=utf8 + +# GZip import support (optional) +DB_IMPORT_GZIP=false + +# SQL or Gzip import file (required) +# If `DB_IMPORT_GZIP` is `true` then file name should be something like `database_name.sql.gz` +DB_IMPORT_FILE_PATH=database_name.sql + +# Database settings (required) +DB_NAME="" +DB_USERNAME="" +DB_PASSWORD="" + +# Additional arguments (optional) +DB_ARGS= diff --git a/8.0/scripts/___mysqlexport.sh b/8.0/scripts/___mysqlexport.sh new file mode 100755 index 0000000..de82f15 --- /dev/null +++ b/8.0/scripts/___mysqlexport.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +START=`date +%s` + +XDB_PROTO="$DB_PROTOCOL" +XDB_HOST="$DB_HOST" +XDB_PORT="$DB_PORT" +XDB_DEFAULT_CHARACTER_SET="$DB_DEFAULT_CHARACTER_SET" +XDB_EXPORT_FILE="$DB_EXPORT_FILE_PATH" +XDB_EXPORT_GZIP="$DB_EXPORT_GZIP" +XDB_EXPORT= + +# Required env variables +if [[ -z "$DB_NAME" ]]; then "ERROR: `DB_NAME` env variable is required."; exit 1; fi +if [[ -z "$DB_USERNAME" ]]; then "ERROR: `DB_USERNAME` env variable is required."; exit 1; fi +if [[ -z "$DB_PASSWORD" ]]; then "ERROR: `DB_PASSWORD` env variable is required."; exit 1; fi + +# Optional env variables +if [[ -z "$XDB_PROTO" ]]; then XDB_PROTO="tcp"; fi +if [[ -z "$XDB_HOST" ]]; then XDB_HOST="127.0.0.1"; fi +if [[ -z "$XDB_PORT" ]]; then XDB_PORT="3306"; fi +if [[ -z "$XDB_DEFAULT_CHARACTER_SET" ]]; then XDB_DEFAULT_CHARACTER_SET=utf8; fi +if [[ -z "$DB_EXPORT_FILE_PATH" ]]; then XDB_EXPORT_FILE="./$DB_NAME.sql"; fi +if [[ -n "$XDB_EXPORT_GZIP" ]] && [[ "$XDB_EXPORT_GZIP" = "true" ]]; then + if [[ -z $DB_EXPORT_FILE_PATH ]]; then XDB_EXPORT_FILE="$XDB_EXPORT_FILE.gz"; fi + + XDB_EXPORT="| gzip -c > $XDB_EXPORT_FILE" +else + XDB_EXPORT="> $XDB_EXPORT_FILE" +fi + +DB_PASSWORD=$(echo -n $DB_PASSWORD | sed 's/"/\\"/g') + +CMD="\ +--protocol=$XDB_PROTO \ +--host=$XDB_HOST \ +--port=$XDB_PORT \ +--default-character-set=$XDB_DEFAULT_CHARACTER_SET \ +--user=$DB_USERNAME \ +--password="\"$DB_PASSWORD"\" \ +$DB_ARGS $DB_NAME $XDB_EXPORT" + +echo "MySQL 8 Client - Exporter" +echo "=========================" + +mysqldump --version + +echo +echo "Exporting database \`$DB_NAME\` into a SQL script file..." + +if [[ -n "$XDB_EXPORT_GZIP" ]] && [[ "$XDB_EXPORT_GZIP" = "true" ]]; then + echo "Output file: $XDB_EXPORT_FILE (SQL GZipped)" +else + echo "Output file: $XDB_EXPORT_FILE (SQL Text)" +fi + +OUTPUT=$(eval mysqldump ${CMD} 2>&1) +exitcode=$? + +if [[ $exitcode != 0 ]]; then echo $OUTPUT; exit $exitcode; fi + +# Note: Ugly workaround here because `mysqldump` (unlike `mysql`) doesn't emit a proper exit code even in MySQL v8 +# See https://bugs.mysql.com/bug.php?id=90538 +if echo $OUTPUT | grep -qE '(.*)(mysqldump: Got error:|: eval:)(.*)'; then + echo $OUTPUT + exit 1 +fi + +if [[ ! -z "$OUTPUT" ]]; then echo $OUTPUT; fi; + +FILE_SIZE=$(du -sh $XDB_EXPORT_FILE | cut -f1) + +END=`date +%s` +RUNTIME=$((END-START)) + +echo "Database \`$DB_NAME\` was exported on ${RUNTIME}s successfully!" + +if [[ -n "$XDB_EXPORT_GZIP" ]] && [[ "$XDB_EXPORT_GZIP" = "true" ]]; then + echo "File exported: $XDB_EXPORT_FILE ($FILE_SIZE / SQL GZipped)" +else + echo "File exported: $XDB_EXPORT_FILE ($FILE_SIZE / SQL Text)" +fi diff --git a/8.0/scripts/___mysqlimport.sh b/8.0/scripts/___mysqlimport.sh new file mode 100755 index 0000000..466ebf9 --- /dev/null +++ b/8.0/scripts/___mysqlimport.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +set -e + +START=`date +%s` + +XDB_PROTO="$DB_PROTOCOL" +XDB_HOST="$DB_HOST" +XDB_PORT="$DB_PORT" +XDB_DEFAULT_CHARACTER_SET="$DB_DEFAULT_CHARACTER_SET" +XDB_IMPORT_FILE="$DB_IMPORT_FILE_PATH" +XDB_IMPORT_GZIP="$DB_IMPORT_GZIP" +XDB_IMPORT= + +# Required env variables +if [[ -z "$DB_NAME" ]]; then "ERROR: `DB_NAME` env variable is required."; exit 1; fi +if [[ -z "$DB_USERNAME" ]]; then "ERROR: `DB_USERNAME` env variable is required."; exit 1; fi +if [[ -z "$DB_PASSWORD" ]]; then "ERROR: `DB_PASSWORD` env variable is required."; exit 1; fi +if [[ -z "$DB_IMPORT_FILE_PATH" ]]; then "ERROR: `DB_IMPORT_FILE_PATH` env variable is required."; exit 1; fi + +# Optional env variables +if [[ -z "$XDB_PROTO" ]]; then XDB_PROTO="tcp"; fi +if [[ -z "$XDB_HOST" ]]; then XDB_HOST="127.0.0.1"; fi +if [[ -z "$XDB_PORT" ]]; then XDB_PORT="3306"; fi +if [[ -z "$XDB_DEFAULT_CHARACTER_SET" ]]; then XDB_DEFAULT_CHARACTER_SET=utf8; fi +if [[ -n "$XDB_IMPORT_GZIP" ]] && [[ "$XDB_IMPORT_GZIP" = "true" ]]; then + XDB_IMPORT="gzip -dc $XDB_IMPORT_FILE |" + XDB_IMPORT_FILE= +else + XDB_IMPORT_FILE="< $XDB_IMPORT_FILE" +fi + +DB_PASSWORD=$(echo -n $DB_PASSWORD | sed 's/"/\\"/g') + +CMD="\ +--protocol=$XDB_PROTO \ +--host=$XDB_HOST \ +--port=$XDB_PORT \ +--default-character-set=$XDB_DEFAULT_CHARACTER_SET \ +--user=$DB_USERNAME \ +--password="\"$DB_PASSWORD"\" \ +$DB_ARGS $DB_NAME $XDB_IMPORT_FILE" + +echo "MySQL 8 Client - Importer" +echo "=========================" + +mysql --version + +FILE_SIZE=$(du -sh $DB_IMPORT_FILE_PATH | cut -f1) + +echo +echo "Importing a SQL script file into database \`$DB_NAME\`..." + +if [[ -n "$XDB_IMPORT_GZIP" ]] && [[ "$XDB_IMPORT_GZIP" = "true" ]]; then + echo "Input file: $DB_IMPORT_FILE_PATH ($FILE_SIZE / SQL GZipped)" +else + echo "Input file: $DB_IMPORT_FILE_PATH ($FILE_SIZE / SQL Text)" +fi + +eval "${XDB_IMPORT}mysql ${CMD}" + +END=`date +%s` +RUNTIME=$((END-START)) + +echo "Database \`$DB_NAME\` was imported on ${RUNTIME}s successfully!" diff --git a/8.0/scripts/mysql_exporter b/8.0/scripts/mysql_exporter new file mode 100755 index 0000000..7242d60 --- /dev/null +++ b/8.0/scripts/mysql_exporter @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +FILE_ENV=$1 + +if [[ -z "$FILE_ENV" ]]; then + ___mysqlexport.sh + exit 0 +fi + +if [[ -f $FILE_ENV ]]; then + enve -f $FILE_ENV ___mysqlexport.sh +else + echo "ERROR: env file \`$FILE_ENV\` was not found" + exit 1 +fi diff --git a/8.0/scripts/mysql_importer b/8.0/scripts/mysql_importer new file mode 100755 index 0000000..7b53f34 --- /dev/null +++ b/8.0/scripts/mysql_importer @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +FILE_ENV=$1 + +if [[ -z "$FILE_ENV" ]]; then + ___mysqlimport.sh + exit 0 +fi + +if [[ -f $FILE_ENV ]]; then + enve -f $FILE_ENV ___mysqlimport.sh +else + echo "ERROR: env file \`$FILE_ENV\` was not found" + exit 1 +fi diff --git a/Makefile b/Makefile index 86782b0..32d803a 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,13 @@ build: @docker build -t mysql-client:latest -f 8.0/Dockerfile . .PHONY: build +HOME_USER ?= $(shell echo $$(id -u $$USER):$$(id -g $$USER)) + export: @docker run --rm -it \ --user $(HOME_USER) \ - --name mysql-client \ --volume $(PWD):/home/mysql/sample \ - --network mysql-net \ + --network mysql8_net \ --workdir /home/mysql/sample \ mysql-client:latest \ mysql_exporter export.env @@ -16,9 +17,8 @@ export: import: @docker run --rm -it \ --user $(HOME_USER) \ - --name mysql-client \ --volume $(PWD):/home/mysql/sample \ - --network mysql-net \ + --network mysql8_net \ --workdir /home/mysql/sample \ mysql-client:latest \ mysql_importer import.env diff --git a/README.md b/README.md index c91ae62..e024524 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ > [MySQL 8 client](https://dev.mysql.com/doc/refman/8.0/en/programs-client.html) for export and import databases easily using Docker. +This is a __Linux Docker image__ using latest __Debian [11-slim](https://hub.docker.com/_/debian?tab=tags&page=1&name=11-slim)__ ([Bullseye](https://www.debian.org/News/2021/20210814)). + 🐳 View on [Docker Hub](https://hub.docker.com/r/joseluisq/mysql-client/) ## MySQL 8 Client programs @@ -23,6 +25,159 @@ mysqlshow For more details see official [MySQL 8 Client Programs](https://dev.mysql.com/doc/refman/8.0/en/programs-client.html) documentation. +## User privileges + +- Default user (unprivileged) is `mysql`. +- `mysql` home directory is located at `/home/mysql`. +- If you want a full privileged user try `root`. E.g append a `--user root` argument to `docker run`. + +## Exporter + +`mysql_exporter` is a custom tool which exports a database script using `mysqldump`. Additionally it support gzip compression. +It can be configured via environment variables or using `.env` file. + +### Setup via environment variables + +```env +# Connection settings (optional) +DB_PROTOCOL=tcp +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DEFAULT_CHARACTER_SET=utf8 + +# GZip export file (optional) +DB_EXPORT_GZIP=false + +# SQL or Gzip export file (optional). +# If `DB_IMPORT_GZIP` is `true` then file name should be `database_name.sql.gz` +DB_EXPORT_FILE_PATH=database_name.sql + +# Database settings (required) +DB_NAME="" +DB_USERNAME="" +DB_PASSWORD="" + +# Additional arguments (optional) +DB_ARGS= +``` + +**Notes:** + +- `DB_EXPORT_GZIP=true`: Compress the sql file using Gzip (optional). If `false` or not defined then the exported file will be a `.sql` file. +- `DB_ARGS`: can be used in order to pass more `mysqldump` arguments (optional). +- An `.env` example file can be found at [./8.0/env/mysql_exporter.env](./8.0/env/mysql_exporter.env) + +### Export a database using a Docker container + +The following Docker commands create a container in order to export a database and then remove such container automatically. + +Note that `mysql_exporter` supports environment variables or a `.env` file can be passed as an argument. + +```sh +docker run --rm -it \ + --volume $PWD:/home/mysql/sample \ + --user $(id -u $USER):$(id -g $USER) \ + --workdir /home/mysql/sample \ + joseluisq/mysql-client:8 \ + mysql_exporter production.env + +# MySQL 8 Client - Exporter +# ========================= +# mysqldump Ver 8.0.28 for Linux on x86_64 (MySQL Community Server - GPL) + +# Exporting database `mydb` into a SQL script file... +# Output file: database_name.sql (SQL Text) +# mysqldump: [Warning] Using a password on the command line interface can be insecure. +# Database `mydb` was exported on 0s successfully! +# File exported: database_name.sql (4.0K / SQL Text) +``` + +__Notes:__ + +- `--volume $PWD:/home/mysql/sample` specificy a [bind mount directory](https://docs.docker.com/storage/bind-mounts/) from host to container. +- `$PWD` is just a example host working directory. Use your own path. +- `/home/mysql/` is default home directory user (optional). View [User privileges](#user-privileges) section above. +- `/home/mysql/sample` is a container directory that Docker will create for us. +- `--workdir /home/mysql/sample` specificy the working directory used by default inside the container. +- `production.env` is a custom env file path with the corresponding environment variables passed as argument. That file shoud available in your host working directory. E.g `$PWD` in my case. + +### Export a database using a Docker Compose file + +```yaml +version: "3.3" + +services: + exporter: + image: joseluisq/mysql-client:8 + env_file: .env + command: mysql_exporter + working_dir: /home/mysql/sample + volumes: + - ./:/home/mysql/sample + networks: + - default +``` + +## Importer + +`mysql_importer` is a custom tool which imports a SQL script file (text or Gzip) using `mysql` command. +It can be configured via environment variables or using `.env` file. + +### Setup via environment variables + +```env +# Connection settings (optional) +DB_PROTOCOL=tcp +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DEFAULT_CHARACTER_SET=utf8 + +# GZip import support (optional) +DB_IMPORT_GZIP=false + +# SQL or Gzip import file (required) +# If `DB_IMPORT_GZIP` is `true` then file name should be `database_name.sql.gz` +DB_IMPORT_FILE_PATH=database_name.sql + +# Database settings (required) +DB_NAME="" +DB_USERNAME="" +DB_PASSWORD="" + +# Additional arguments (optional) +DB_ARGS= +``` + +### Import a SQL script via a Docker container + +The following Docker commands create a container in order to import a SQL script file to an specific database and removing the container afterwards. + +Note that `mysql_importer` supports environment variables or a `.env` file can be passed as an argument. + +```sh +docker run --rm -it \ + --volume $PWD:/home/mysql/sample \ + --user $(id -u $USER):$(id -g $USER) \ + --workdir /home/mysql/sample \ + joseluisq/mysql-client:8 \ + mysql_importer production.env + +# MySQL 8 Client - Importer +# ========================= +# mysql Ver 8.0.28 for Linux on x86_64 (MySQL Community Server - GPL) + +# Importing a SQL script file into database `mydb`... +# Input file: database_name.sql (4.0K / SQL Text) +# mysql: [Warning] Using a password on the command line interface can be insecure. +# Database `mydb` was imported on 1s successfully! +``` + +**Notes:** + +- `DB_IMPORT_GZIP=true`: Decompress a dump file using Gzip (optional). If `false` or not defined then the import file will be treated as plain `.sql` file. +- `DB_ARGS`: can be used in order to pass more `mysql` arguments (optional). +- An `.env` example file can be found at [./8.0/env/mysql_importer.env](./8.0/env/mysql_importer.env) + ## Contributions Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in current work by you, as defined in the Apache-2.0 license, shall be dual licensed as described below, without any additional terms or conditions.