Merge pull request #354 from effectivelywild/main

Resolve multiple issues using Azure blobs for remote storage
tiredofit · Jul 2, 2024 · 0fdb447 · 0fdb447
2 parents b9fa7d1 + 0d23c26
commit 0fdb447
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -356,11 +356,14 @@ If `DEFAULT_BACKUP_LOCATION` = `S3` then the following options are used:
 
 If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
-| Parameter                              | Description                                 | Default             | `_FILE` |
-| -------------------------------------- | ------------------------------------------- | ------------------- | ------- |
-| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name. |                     | x       |
-| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.  |                     | x       |
-| `DEFAULT_BLOBXFER_REMOTE_PATH`         | Remote Azure path                           | `/docker-db-backup` | x       |
+| Parameter                              | Description                                                         | Default             | `_FILE` |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
+| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name.                         |                     | x       |
+| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.                          |                     | x       |
+| `DEFAULT_BLOBXFER_REMOTE_PATH`         | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DEFAULT_BLOBXFER_REMOTE_MODE`         | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.
@@ -636,11 +639,14 @@ If `DB01_BACKUP_LOCATION` = `S3` then the following options are used:
 
 If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
-| Parameter                           | Description                                 | Default             | `_FILE` |
-| ----------------------------------- | ------------------------------------------- | ------------------- | ------- |
-| `DB01_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name. |                     | x       |
-| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.  |                     | x       |
-| `DB01_BLOBXFER_REMOTE_PATH`         | Remote Azure path                           | `/docker-db-backup` | x       |
+| Parameter                              | Description                                                         | Default             | `_FILE` |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
+| `DB01_BLOBXFER_STORAGE_ACCOUNT`        | Microsoft Azure Cloud storage account name.                         |                     | x       |
+| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY`    | Microsoft Azure Cloud storage account key.                          |                     | x       |
+| `DB01_BLOBXFER_REMOTE_PATH`            | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DB01_BLOBXFER_REMOTE_MODE`            | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup directory `DB01_BACKUP_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.

diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup
@@ -8,6 +8,7 @@ DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
 DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"}
 DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}
+DEFAULT_BLOBXFER_MODE=${DEFAULT_BLOBXFER_MODE:-"auto"}
 DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"}
 DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"}
 DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"}

diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup
@@ -66,6 +66,7 @@ bootstrap_variables() {
                             DEFAULT_BLOBXFER_STORAGE_ACCOUNT \
                             DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DEFAULT_BLOBXFER_REMOTE_PATH \
+                            DEFAULT_BLOBXFER_MODE \
                             DB"${backup_instance_number}"_AUTH \
                             DB"${backup_instance_number}"_TYPE \
                             DB"${backup_instance_number}"_HOST \
@@ -93,6 +94,7 @@ bootstrap_variables() {
                             DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT \
                             DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \
+                            DB"${backup_instance_number}"_BLOBXFER_MODE \
                             BLOBXFER_STORAGE_ACCOUNT \
                             BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB_HOST \
@@ -199,6 +201,7 @@ bootstrap_variables() {
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT_KEY backup_job_blobxfer_storage_account_key
+        transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_MODE backup_job_blobxfer_mode
         transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum
         transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time
         transform_backup_instance_variable "${backup_instance_number}" COMPRESSION backup_job_compression
@@ -1072,7 +1075,7 @@ cleanup_old_data() {
                         write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
                     else
                         write_log info "Syncing changes via blobxfer"
-                        silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                        silent run_as_user blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
                     fi
                 ;;
                 "file" | "filesystem" )
@@ -1635,16 +1638,16 @@ EOF
                 if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
                     write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
                 else
-                    write_log info "Synchronize local storage from S3 Bucket with blobxfer"
-                    ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete
+                    write_log info "Synchronize local storage from blob container with blobxfer"
+                    ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete
 
                     write_log info "Moving backup to external storage with blobxfer"
                     mkdir -p "${backup_job_filesystem_path}"
                     if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${temporary_directory}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi
 
                     run_as_user mv "${temporary_directory}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
 
-                    silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
+                    silent run_as_user ${play_fair} blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
                     move_exit_code=$?
 
                     if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${temporary_directory}"/"${backup_job_filename}"."${checksum_extension}" ; fi