LPD-68592 Import/export via creating and extracting tar files after image creation to avoid large Docker contexts

.dockerignore

@@ -1 +1,17 @@
-**/.gitkeep
+**/.gitkeep
+.github/
+.gradle/
+build/
+buildSrc/
+data/
+dumps/
+exported_data/
+build.gradle
+gradle.properties
+gradle-local.properties
+gradlew
+gradlew.bat
+README.markdown
+settings.gradle
+source-formatter-suppressions.xml
+source-formatter.properties

build.gradle

@@ -1,5 +1,3 @@
-import com.bmuschko.gradle.docker.tasks.container.DockerCopyFileFromContainer
-
 import com.liferay.docker.workspace.environments.Util
 
 import groovy.json.JsonSlurper
@@ -32,26 +30,76 @@ withZone(ZoneId.systemDefault())
 
 String timestamp = formatter.format(now)
 
-tasks.register("exportContainerData", DockerCopyFileFromContainer) {
-	containerId = "${config.namespace}-data-helper"
-
+tasks.register("exportContainerData") {
 	doFirst {
-		mkdir hostPath.get()
+		String hostPath = projectDir.relativePath(file("exported_data/data_${config.namespace}_${timestamp}"))
 
-		waitForCommand("docker compose create data-helper")
-	}
+		getExistingVolumeNames().each {
+			String volumeName ->
 
-	doLast {
-		updateGradleLocalProperties(["lr.docker.environment.data.directory" : hostPath.get()])
+			String backupFileName = "${volumeName.substring(config.namespace.length() + 1)}.tar"
 
-		logger.lifecycle("\nUpdated gradle-local.properties with the new value:\nlr.docker.environment.data.directory=${hostPath.get()}")
-	}
+			if (backupFileName.equals("dumps.tar")) {
+				return
+			}
+
+			println "Creating backup of volume ${volumeName} in ${hostPath}/${backupFileName}"
+			waitForCommand("docker run --rm -v ${volumeName}:/source -v ${file(hostPath).getAbsolutePath()}:/target busybox:latest tar --create --file=/target/${backupFileName} --directory=/source .")
+			println "Created backup of volume ${volumeName} in ${hostPath}/${backupFileName}"
+		}
 
-	hostPath = provider {
-		"exported_data/data_${config.namespace}_${timestamp}"
+		updateGradleLocalProperties(["lr.docker.environment.data.directory" : hostPath])
+
+		logger.lifecycle("\nUpdated gradle-local.properties with the new value:\nlr.docker.environment.data.directory=${hostPath}")
 	}
+}
+
+tasks.register("importContainerData") {
+	mustRunAfter ":buildDockerImage"
+
+	doFirst {
+		List<String> previousVolumeNames = getExistingVolumeNames()
+
+		waitForCommand("docker compose build")
+		waitForCommand("docker compose create")
+
+		List<String> currentVolumeNames = getExistingVolumeNames()
+
+		project.file(config.dataDirectory).listFiles().each {
+			File backupFile ->
+
+			if (backupFile.isFile() && !backupFile.name.endsWith(".tar")) {
+				return
+			}
+
+			String volumeName = "${config.namespace}_${backupFile.name}"
+
+			if (backupFile.isFile()) {
+				volumeName = volumeName.substring(0, volumeName.length() - 4)
+			}
 
-	remotePath = "/container-data"
+			if (!currentVolumeNames.contains(volumeName)) {
+				return
+			}
+
+			if (previousVolumeNames.contains(volumeName)) {
+				return
+			}
+
+			println "Restoring backup of volume ${volumeName} using ${backupFile.absolutePath}"
+
+			if (backupFile.isFile()) {
+				waitForCommand("docker run --rm -v ${volumeName}:/target -v ${backupFile.absolutePath}:/source.tar busybox:latest tar --extract --file=/source.tar --directory=/target/")
+			}
+			else {
+				waitForCommand("docker run --rm -v ${volumeName}:/target -v ${backupFile.absolutePath}:/source busybox:latest sh -c 'cp --update --recursive /source/* /target/'")
+			}
+
+			println "Restored backup of volume ${volumeName} using ${backupFile.absolutePath}"
+
+			println waitForCommand("docker run --rm -v ${volumeName}:/${volumeName} busybox:latest du -sh /${volumeName}/")
+		}
+	}
 }
 
 tasks.register("importDocumentLibraryStructure") {
@@ -74,37 +122,41 @@ tasks.register("importDocumentLibraryStructure") {
 	}
 }
 
-tasks.register("shareWorkspace", Zip) {
-	archiveFileName = provider {
-		"workspace_${config.namespace}_${timestamp}.zip"
-	}
-
-	destinationDirectory = file 'shared_workspaces'
-
+tasks.register("shareWorkspace") {
 	doLast {
-		logger.lifecycle "\nWorkspace zip: ${archiveFile.get()}\n"
-	}
-
-	exclude ".gitkeep"
-	exclude ".gitignore"
-	exclude ".git"
-	exclude ".gradle"
-	exclude "dumps/*"
-	exclude "shared_workspaces"
-	exclude {
-		FileTreeElement fileTreeElement ->
-
-		fileTreeElement.relativePath.pathString.startsWith("exported_data/") &&
-		!fileTreeElement.relativePath.pathString.startsWith(config.dataDirectory)
-	}
+		String fileListFileName = "shared_workspaces/workspace_${config.namespace}_${timestamp}.txt"
+		String archiveFileName = "shared_workspaces/workspace_${config.namespace}_${timestamp}.7z"
+
+		file(fileListFileName).text = fileTree('.') {
+			exclude ".gitkeep"
+			exclude ".gitignore"
+			exclude ".git"
+			exclude ".gradle"
+			exclude "buildSrc/build"
+			exclude "buildSrc/.gradle"
+			exclude "dumps/*"
+			exclude "shared_workspaces"
+			exclude {
+				FileTreeElement fileTreeElement ->
+
+				fileTreeElement.relativePath.pathString.startsWith("exported_data/") &&
+				!fileTreeElement.relativePath.pathString.startsWith(config.dataDirectory)
+			}
 
-	from '.'
+			include "${config.dataDirectory}/**"
+			include "**/*"
+		}.files.collect {
+			File file ->
 
-	include "${config.dataDirectory}/**"
-	include "**/*"
+			project.relativePath(file)
+		}.join("\n")
 
-	outputs.upToDateWhen {
-		false
+		try {
+			waitForCommand("7z a -m0=lzma2 -mx=1 -bsp2 ${archiveFileName} -i@${fileListFileName}")
+		}
+		finally {
+			file(fileListFileName).delete()
+		}
 	}
 }
 
@@ -163,6 +215,7 @@ composeUp {
 	}
 
 	dependsOn ":buildDockerImage"
+	dependsOn ":importContainerData"
 	dependsOn ":importDatabaseDumps"
 
 	doFirst {

compose-recipes/db2/service.db2.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   db2:/container-data/db2
     database:
         build:
             args:

compose-recipes/elasticsearch/service.elasticsearch.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   elasticsearch:/container-data/elasticsearch
     elasticsearch:
         build: ./compose-recipes/elasticsearch
         container_name: ${NAMESPACE}-elasticsearch

compose-recipes/liferay/service.liferay.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   liferay:/container-data/liferay
     liferay:
         container_name: ${NAMESPACE}-liferay
         env_file:

compose-recipes/mariadb/service.mariadb.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   mariadb:/container-data/mariadb
     database:
         build:
             args:

compose-recipes/mysql/service.mysql.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   mysql:/container-data/mysql
     database:
         build:
             args:

compose-recipes/postgres/service.postgres.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   postgres:/container-data/postgres
     database:
         build:
             args:

compose-recipes/sqlserver/service.sqlserver.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   sqlserver:/container-data/sqlserver
     database:
         build:
             args:

compose-recipes/webserver-http/service.webserver_http.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   webserver-http:/container-data/webserver-http
     webserver:
         build: ./compose-recipes/webserver-http
         container_name: ${NAMESPACE}-webserver-http

compose-recipes/webserver-https/service.webserver_https.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   webserver-https:/container-data/webserver-https
     webserver:
         build: ./compose-recipes/webserver-https
         container_name: ${NAMESPACE}-webserver-https

docker-compose.yaml

@@ -1,11 +1,3 @@
 name: ${NAMESPACE}
-services:
-    data-helper:
-        build:
-            args:
-                DATA_DIRECTORY: ${DATA_DIRECTORY}
-            context: .
-            dockerfile: Dockerfile.helper
-        container_name: ${NAMESPACE}-data-helper
 volumes:
     dumps: