Merge pull request #69 from essentialkaos/develop

Version 0.2.0

.github/dependabot.yml

@@ -6,8 +6,8 @@ updates:
     target-branch: "develop"
     schedule:
       interval: "daily"
-      timezone: "Europe/London"
-      time: "03:00"
+      timezone: "Etc/UTC"
+      time: "07:00"
     labels:
       - "PR • MAINTENANCE"
     assignees:
@@ -26,8 +26,8 @@ updates:
     target-branch: "develop"
     schedule:
       interval: "daily"
-      timezone: "Europe/London"
-      time: "04:00"
+      timezone: "Etc/UTC"
+      time: "08:00"
     labels:
       - "PR • MAINTENANCE"
     assignees:

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        go: [ '1.21.x', '1.22.x' ]
+        go: [ '1.22.x', '1.23.x' ]
 
     steps:
       - name: Checkout

Makefile

@@ -1,6 +1,6 @@
 ################################################################################
 
-# This Makefile generated by GoMakeGen 3.0.5 using next command:
+# This Makefile generated by GoMakeGen 3.1.0 using next command:
 # gomakegen --mod .
 #
 # More info: https://kaos.sh/gomakegen
@@ -11,6 +11,10 @@ ifdef VERBOSE ## Print verbose information (Flag)
 VERBOSE_FLAG = -v
 endif
 
+ifdef PROXY ## Force proxy usage for downloading dependencies (Flag)
+export GOPROXY=https://proxy.golang.org/cached-only,direct
+endif
+
 COMPAT ?= 1.19
 MAKEDIR = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 GITREV ?= $(shell test -s $(MAKEDIR)/.git && git rev-parse --short HEAD)
@@ -109,6 +113,6 @@ help: ## Show this info
 		| sed 's/ifdef //' \
 		| awk 'BEGIN {FS = " .*?## "}; {printf "  \033[32m%-11s\033[0m  %s\n", $$1, $$2}'
 	@echo -e ''
-	@echo -e '\033[90mGenerated by GoMakeGen 3.0.5\033[0m\n'
+	@echo -e '\033[90mGenerated by GoMakeGen 3.1.0\033[0m\n'
 
 ################################################################################

README.md

@@ -2,6 +2,7 @@
 
 <p align="center">
   <a href="https://kaos.sh/l/atlassian-cloud-backuper"><img src="https://kaos.sh/l/c742a6f5789762426f97.svg" alt="Code Climate Maintainability" /></a>
+  <a href="https://kaos.sh/y/atlassian-cloud-backuper"><img src="https://kaos.sh/y/1fc3510176ab43c6a8cd1f8faf0ad309.svg" alt="Codacy badge" /></a>
   <a href="https://kaos.sh/w/atlassian-cloud-backuper/ci"><img src="https://kaos.sh/w/atlassian-cloud-backuper/ci.svg" alt="GitHub Actions CI Status" /></a>
   <a href="https://kaos.sh/w/atlassian-cloud-backuper/codeql"><img src="https://kaos.sh/w/atlassian-cloud-backuper/codeql.svg" alt="GitHub Actions CodeQL Status" /></a>
   <a href="#license"><img src=".github/images/license.svg"/></a>

app/app.go

@@ -42,7 +42,7 @@ import (
 // Basic utility info
 const (
 	APP  = "Atlassian Cloud Backuper"
-	VER  = "0.1.0"
+	VER  = "0.2.0"
 	DESC = "Tool for backuping Atlassian cloud services (Jira and Confluence)"
 )
 
@@ -53,6 +53,7 @@ const (
 	OPT_CONFIG      = "c:config"
 	OPT_INTERACTIVE = "I:interactive"
 	OPT_SERVER      = "S:server"
+	OPT_FORCE       = "F:force"
 	OPT_NO_COLOR    = "nc:no-color"
 	OPT_HELP        = "h:help"
 	OPT_VER         = "v:version"
@@ -84,6 +85,7 @@ const (
 	STORAGE_S3_SECRET_KEY          = "storage-s3:secret-key"
 	STORAGE_S3_BUCKET              = "storage-s3:bucket"
 	STORAGE_S3_PATH                = "storage-s3:path"
+	STORAGE_S3_PART_SIZE           = "storage-s3:part-size"
 	JIRA_OUTPUT_FILE               = "jira:output-file"
 	JIRA_INCLUDE_ATTACHMENTS       = "jira:include-attachments"
 	JIRA_CLOUD_FORMAT              = "jira:cloud-format"
@@ -116,6 +118,7 @@ const (
 // optMap contains information about all supported options
 var optMap = options.Map{
 	OPT_CONFIG:      {Value: "/etc/atlassian-cloud-backuper.knf"},
+	OPT_FORCE:       {Type: options.BOOL},
 	OPT_INTERACTIVE: {Type: options.BOOL},
 	OPT_SERVER:      {Type: options.BOOL},
 	OPT_NO_COLOR:    {Type: options.BOOL},
@@ -249,7 +252,7 @@ func addExtraOptions(m options.Map) {
 		STORAGE_SFTP_HOST, STORAGE_SFTP_USER, STORAGE_SFTP_KEY,
 		STORAGE_SFTP_PATH, STORAGE_SFTP_MODE,
 		STORAGE_S3_HOST, STORAGE_S3_ACCESS_KEY, STORAGE_S3_SECRET_KEY,
-		STORAGE_S3_BUCKET, STORAGE_S3_PATH,
+		STORAGE_S3_BUCKET, STORAGE_S3_PATH, STORAGE_S3_PART_SIZE,
 		JIRA_OUTPUT_FILE, JIRA_INCLUDE_ATTACHMENTS, JIRA_CLOUD_FORMAT,
 		CONFLUENCE_OUTPUT_FILE, CONFLUENCE_INCLUDE_ATTACHMENTS, CONFLUENCE_CLOUD_FORMAT,
 		TEMP_DIR,
@@ -284,7 +287,7 @@ func loadConfig() error {
 			STORAGE_SFTP_HOST, STORAGE_SFTP_USER, STORAGE_SFTP_KEY,
 			STORAGE_SFTP_PATH, STORAGE_SFTP_MODE,
 			STORAGE_S3_HOST, STORAGE_S3_REGION, STORAGE_S3_ACCESS_KEY,
-			STORAGE_S3_SECRET_KEY, STORAGE_S3_BUCKET, STORAGE_S3_PATH,
+			STORAGE_S3_SECRET_KEY, STORAGE_S3_BUCKET, STORAGE_S3_PATH, STORAGE_S3_PART_SIZE,
 			JIRA_OUTPUT_FILE, JIRA_INCLUDE_ATTACHMENTS, JIRA_CLOUD_FORMAT,
 			CONFLUENCE_OUTPUT_FILE, CONFLUENCE_INCLUDE_ATTACHMENTS, CONFLUENCE_CLOUD_FORMAT,
 			TEMP_DIR,
@@ -334,6 +337,8 @@ func validateConfig() error {
 			&knf.Validator{STORAGE_S3_ACCESS_KEY, knfv.Set, nil},
 			&knf.Validator{STORAGE_S3_SECRET_KEY, knfv.Set, nil},
 			&knf.Validator{STORAGE_S3_BUCKET, knfv.Set, nil},
+			&knf.Validator{STORAGE_S3_PART_SIZE, knfv.Greater, 5},
+			&knf.Validator{STORAGE_S3_PART_SIZE, knfv.Less, 5_000},
 		)
 	}
 
@@ -498,6 +503,7 @@ func genUsage(section string) *usage.Info {
 	info.AddOption(OPT_CONFIG, "Path to configuration file", "file")
 	info.AddOption(OPT_INTERACTIVE, "Interactive mode")
 	info.AddOption(OPT_SERVER, "Server mode")
+	info.AddOption(OPT_FORCE, "Force backup generation")
 	info.AddOption(OPT_NO_COLOR, "Disable colors in output")
 	info.AddOption(OPT_HELP, "Show this help message")
 	info.AddOption(OPT_VER, "Show version")
@@ -524,6 +530,7 @@ func genUsage(section string) *usage.Info {
 		addUnitedOption(info, STORAGE_S3_SECRET_KEY, "S3 access secret key", "key")
 		addUnitedOption(info, STORAGE_S3_BUCKET, "S3 bucket", "name")
 		addUnitedOption(info, STORAGE_S3_PATH, "Path for backups", "path")
+		addUnitedOption(info, STORAGE_S3_PART_SIZE, "Uploading part size (in MB)", "num")
 		addUnitedOption(info, JIRA_OUTPUT_FILE, "Jira backup output file name template", "template")
 		addUnitedOption(info, JIRA_INCLUDE_ATTACHMENTS, "Include attachments to Jira backup", "yes/no")
 		addUnitedOption(info, JIRA_CLOUD_FORMAT, "Create Jira backup for Cloud", "yes/no")

app/basic.go

@@ -73,7 +73,7 @@ func startApp(args options.Arguments) error {
 
 	tmpFile := path.Join(tmpDir, outputFileName)
 
-	err = bkpr.Backup(tmpFile)
+	err = bkpr.Backup(tmpFile, options.GetB(OPT_FORCE))
 
 	if err != nil {
 		spinner.Done(false)

app/common.go

@@ -104,9 +104,9 @@ func getUploader(target string) (uploader.Uploader, error) {
 	switch strings.ToLower(knfu.GetS(STORAGE_TYPE)) {
 	case STORAGE_FS:
 		return fs.NewUploader(&fs.Config{
+			Secret: secret,
 			Path:   path.Join(knfu.GetS(STORAGE_FS_PATH), target),
 			Mode:   knfu.GetM(STORAGE_FS_MODE, 0600),
-			Secret: secret,
 		})
 
 	case STORAGE_SFTP:
@@ -117,23 +117,24 @@ func getUploader(target string) (uploader.Uploader, error) {
 		}
 
 		return sftp.NewUploader(&sftp.Config{
+			Secret: secret,
 			Host:   knfu.GetS(STORAGE_SFTP_HOST),
 			User:   knfu.GetS(STORAGE_SFTP_USER),
 			Key:    keyData,
 			Path:   path.Join(knfu.GetS(STORAGE_SFTP_PATH), target),
 			Mode:   knfu.GetM(STORAGE_SFTP_MODE, 0600),
-			Secret: secret,
 		})
 
 	case STORAGE_S3:
 		return s3.NewUploader(&s3.Config{
+			Secret:      secret,
 			Host:        knfu.GetS(STORAGE_S3_HOST),
 			Region:      knfu.GetS(STORAGE_S3_REGION),
 			AccessKeyID: knfu.GetS(STORAGE_S3_ACCESS_KEY),
 			SecretKey:   knfu.GetS(STORAGE_S3_SECRET_KEY),
 			Bucket:      knfu.GetS(STORAGE_S3_BUCKET),
 			Path:        path.Join(knfu.GetS(STORAGE_S3_PATH), target),
-			Secret:      secret,
+			PartSize:    knfu.GetI64(STORAGE_S3_PART_SIZE, 32),
 		})
 	}
 

app/server.go

@@ -58,6 +58,7 @@ func createBackupHandler(rw http.ResponseWriter, r *http.Request) {
 
 	target := strings.ToLower(r.URL.Query().Get("target"))
 	token := r.URL.Query().Get("token")
+	force := r.URL.Query().Get("force") != ""
 
 	err := validateRequestQuery(target, token)
 
@@ -75,7 +76,7 @@ func createBackupHandler(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	taskID, err := bkpr.Start()
+	taskID, err := bkpr.Start(force)
 
 	if err != nil {
 		log.Error("Can't create backup: %v", err)

backuper/backuper.go

@@ -28,13 +28,13 @@ const (
 // Backuper is generic backuper interface
 type Backuper interface {
 	// Backup runs backup process
-	Backup(outputFile string) error
+	Backup(outputFile string, force bool) error
 
 	// SetDispatcher sets events dispatcher
 	SetDispatcher(d *events.Dispatcher)
 
 	// Start creates task for backuping data
-	Start() (string, error)
+	Start(force bool) (string, error)
 
 	// Progress monitors backup creation progress
 	Progress(taskID string) (string, error)

backuper/confluence/confluence-backuper.go

@@ -74,8 +74,8 @@ func (b *ConfluenceBackuper) SetDispatcher(d *events.Dispatcher) {
 }
 
 // Backup starts backup process
-func (b *ConfluenceBackuper) Backup(outputFile string) error {
-	_, err := b.Start()
+func (b *ConfluenceBackuper) Backup(outputFile string, force bool) error {
+	_, err := b.Start(force)
 
 	if err != nil {
 		return err
@@ -91,13 +91,13 @@ func (b *ConfluenceBackuper) Backup(outputFile string) error {
 }
 
 // Start creates task for backuping data
-func (b *ConfluenceBackuper) Start() (string, error) {
+func (b *ConfluenceBackuper) Start(force bool) (string, error) {
 	log.Info("Starting Confluence backup process for account %s…", b.config.Account)
 	log.Info("Checking for existing backup task…")
 
 	info, _ := b.getBackupProgress()
 
-	if info != nil && !info.IsOutdated {
+	if !force && info != nil && !info.IsOutdated {
 		log.Info(
 			"Found previously created backup task",
 			log.F{"backup-status", info.CurrentStatus},
@@ -107,7 +107,11 @@ func (b *ConfluenceBackuper) Start() (string, error) {
 			log.F{"backup-outdated", info.IsOutdated},
 		)
 	} else {
-		log.Info("No previously created backup task or task is outdated, starting new backup…")
+		if force {
+			log.Info("Starting new backup…")
+		} else {
+			log.Info("No previously created backup task or task is outdated, starting new backup…")
+		}
 
 		err := b.startBackup()
 

backuper/jira/jira-backuper.go

@@ -75,8 +75,8 @@ func (b *JiraBackuper) SetDispatcher(d *events.Dispatcher) {
 }
 
 // Backup starts backup process
-func (b *JiraBackuper) Backup(outputFile string) error {
-	backupTaskID, err := b.Start()
+func (b *JiraBackuper) Backup(outputFile string, force bool) error {
+	backupTaskID, err := b.Start(force)
 
 	if err != nil {
 		return err
@@ -92,20 +92,25 @@ func (b *JiraBackuper) Backup(outputFile string) error {
 }
 
 // Start creates task for backuping data
-func (b *JiraBackuper) Start() (string, error) {
+func (b *JiraBackuper) Start(force bool) (string, error) {
 	var err error
 	var backupTaskID string
 
 	log.Info("Starting Jira backup process for account %s…", b.config.Account)
-	log.Info("Checking for existing backup task…")
 
-	backupTaskID, _ = b.getLastTaskID()
+	if !force {
+		log.Info("Checking for existing backup task…")
 
-	if backupTaskID != "" {
-		log.Info("Found previously created backup task with ID %s", backupTaskID)
+		backupTaskID, _ = b.getLastTaskID()
+
+		if backupTaskID == "" {
+			log.Info("No previously created task found, starting new backup…")
+		}
 	} else {
-		log.Info("No previously created task found, starting new backup…")
+		log.Info("Starting new backup…")
+	}
 
+	if backupTaskID == "" {
 		backupTaskID, err = b.startBackup()
 
 		if err != nil {

common/atlassian-cloud-backuper-container.knf

@@ -73,6 +73,9 @@
   # Path to directory with backups
   path:
 
+  # Part size in MB (default: 32)
+  part-size: 32
+
 [jira]
 
   # Backup file name with date tags (default: jira-backup-%Y-%m-%d.zip)

common/atlassian-cloud-backuper.knf

@@ -73,6 +73,9 @@
   # Path to directory with backups
   path:
 
+  # Part size in MB (default: 32)
+  part-size: 32
+
 [jira]
 
   # Backup file name with date tags (default: jira-backup-%Y-%m-%d.zip)

common/atlassian-cloud-backuper.spec

@@ -10,7 +10,7 @@
 
 Summary:        Tool for backuping Atlassian cloud services
 Name:           atlassian-cloud-backuper
-Version:        0.1.0
+Version:        0.2.0
 Release:        0%{?dist}
 Group:          Applications/System
 License:        Apache License, Version 2.0
@@ -114,7 +114,9 @@ rm -rf %{buildroot}
 
 %changelog
 * Tue Jul 23 2024 Anton Novojilov <andy@essentialkaos.com> - 0.1.0-0
+- Added option (-F/--force) and query param (force) to force backup creation
 - Added data encryption feature
+- Added multipart uploading to S3 storage
 - Added server mode
 - Code refactoring
 - Dependencies update