[PSL-1185] split file using 7z (#879)

* [PSL-1185] split file using 7z

* [PSL-1185] updates

Author: mateeullahmalik

common/storage/ticketstore/store.go

@@ -3,10 +3,11 @@ package ticketstore
 import (
 	"context"
 	"fmt"
+	"path/filepath"
+
 	"github.com/jmoiron/sqlx"
 	"github.com/pastelnetwork/gonode/common/configurer"
 	"github.com/pastelnetwork/gonode/common/log"
-	"path/filepath"
 )
 
 const createFilesTable string = `

mixins/go.mod

@@ -25,12 +25,14 @@ require (
 	github.com/fogleman/gg v1.3.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/kolesa-team/go-webp v1.0.4 // indirect
 	github.com/makiuchi-d/gozxing v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

walletnode/api/config.go

@@ -7,10 +7,11 @@ const (
 
 // Config contains settings of the Pastel client.
 type Config struct {
-	Hostname       string `mapstructure:"hostname" json:"hostname,omitempty"`
-	Port           int    `mapstructure:"port" json:"port,omitempty"`
-	Swagger        bool   `mapstructure:"swagger" json:"swagger,omitempty"`
-	StaticFilesDir string `mapstructure:"static_files_dir" json:"static_files_dir,omitempty"`
+	Hostname        string `mapstructure:"hostname" json:"hostname,omitempty"`
+	Port            int    `mapstructure:"port" json:"port,omitempty"`
+	Swagger         bool   `mapstructure:"swagger" json:"swagger,omitempty"`
+	StaticFilesDir  string `mapstructure:"static_files_dir" json:"static_files_dir,omitempty"`
+	CascadeFilesDir string `mapstructure:"cascade_files_dir" json:"cascade_files_dir,omitempty"`
 }
 
 // NewConfig returns a new Config instance.

walletnode/api/services/cascade.go

@@ -67,12 +67,6 @@ func (service *CascadeAPIHandler) UploadAsset(ctx context.Context, p *cascade.Up
 		return nil, cascade.MakeBadRequest(errors.New("file not specified"))
 	}
 
-	fileSize := utils.GetFileSizeInMB(p.Bytes)
-	if fileSize > maxFileSize {
-		log.WithError(err).Error("file size exceeds than 350Mb, please use V2 endpoint for uploading the file")
-		return nil, cascade.MakeInternalServerError(err)
-	}
-
 	id, expiry, err := service.register.StoreFile(ctx, p.Filename)
 	if err != nil {
 		log.WithError(err).Error("error storing File")

walletnode/api/services/common.go

@@ -11,7 +11,8 @@ var randIDGen = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
 
 // Config represents a config for the common service.
 type Config struct {
-	StaticFilesDir string `mapstructure:"static_files_dir" json:"static_files_dir,omitempty"`
+	StaticFilesDir  string `mapstructure:"static_files_dir" json:"static_files_dir,omitempty"`
+	CascadeFilesDir string `mapstructure:"cascade_files_dir" json:"cascade_files_dir,omitempty"`
 }
 
 // Common represents common service.

walletnode/api/services/multipart.go

@@ -3,14 +3,19 @@ package services
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/pastelnetwork/gonode/walletnode/api/gen/cascade"
+	"github.com/pastelnetwork/gonode/walletnode/services/common"
 
+	"github.com/pastelnetwork/gonode/common/random"
 	"github.com/pastelnetwork/gonode/common/storage/files"
 	"github.com/pastelnetwork/gonode/walletnode/api/gen/sense"
 
@@ -74,10 +79,10 @@ func CascadeUploadAssetDecoderFunc(ctx context.Context, service *CascadeAPIHandl
 	return func(reader *multipart.Reader, p **cascade.UploadAssetPayload) error {
 		var res cascade.UploadAssetPayload
 
-		filename, errType, err := handleUploadImage(ctx, reader, service.register.ImageHandler.FileStorage, false)
+		filename, err := handleUploadFile(ctx, reader, service.config.CascadeFilesDir, true)
 		if err != nil {
 			return &goa.ServiceError{
-				Name:    errType,
+				Name:    "",
 				ID:      goa.NewErrorID(),
 				Message: err.Error(),
 			}
@@ -147,3 +152,75 @@ func handleUploadImage(ctx context.Context, reader *multipart.Reader, storage *f
 
 	return filename, "", nil
 }
+
+const (
+	chunkSize = 300 * 1024 * 1024 // 300 MB
+)
+
+func handleUploadFile(ctx context.Context, reader *multipart.Reader, baseDir string, putMaxCap bool) (string, error) {
+	id, err := random.String(8, random.Base62Chars)
+	if err != nil {
+		return "", err
+	}
+
+	dirPath := filepath.Join(baseDir, id)
+
+	// Create the directory
+	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
+		return "", errors.New("could not create directory: " + err.Error())
+	}
+
+	var filename, outputFilePath string
+	var fileSize int64
+
+	for {
+		part, err := reader.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", errors.New("could not read next part: " + err.Error())
+		}
+
+		if part.FormName() != imagePartName {
+			continue
+		}
+
+		filename = part.FileName()
+		fmt.Printf("Upload image %q\n", filename)
+		outputFilePath = filepath.Join(dirPath, filename)
+		outputFile, err := os.Create(outputFilePath)
+		if err != nil {
+			return "", errors.New("could not create file: " + err.Error())
+		}
+
+		n, err := io.Copy(outputFile, part)
+		fileSize += n
+		outputFile.Close()
+
+		if err != nil {
+			return "", errors.New("error writing file: " + err.Error())
+		}
+		break // Assuming single file upload for simplicity
+	}
+
+	fs := common.FileSplitter{PartSizeMB: 300}
+	if fileSize > chunkSize {
+		if putMaxCap {
+			return "", errors.New("file size exceeds the maximum allowed size - Please use API V2 to upload large files")
+		}
+
+		// Use 7z to split the file
+		err := fs.SplitFile(outputFilePath)
+		if err != nil {
+			return "", err
+		}
+		// Remove the original file after splitting
+		if err := os.Remove(outputFilePath); err != nil {
+			return "", errors.New("could not remove original file: " + err.Error())
+		}
+	}
+
+	fmt.Printf("Uploaded image to %q\n", dirPath)
+	return id, nil
+}

walletnode/cmd/app.go

@@ -3,12 +3,13 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"github.com/pastelnetwork/gonode/common/storage/ticketstore"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
 
+	"github.com/pastelnetwork/gonode/common/storage/ticketstore"
+
 	"github.com/pastelnetwork/gonode/common/cli"
 	"github.com/pastelnetwork/gonode/common/configurer"
 	"github.com/pastelnetwork/gonode/common/errors"
@@ -51,6 +52,7 @@ var (
 	defaultPastelConfigFile = filepath.Join(defaultPath, "pastel.conf")
 	defaultRqFilesDir       = filepath.Join(defaultPath, rqFilesDir)
 	defaultStaticFilesDir   = filepath.Join(defaultPath, staticFilesDir)
+	defaultCascadeFilesDir  = filepath.Join(defaultPath, "files")
 )
 
 // NewApp configures our app by parsing command line flags, config files, and setting up logging and temporary directories
@@ -166,6 +168,14 @@ func runApp(ctx context.Context, config *configs.Config) error {
 		}
 	}
 
+	if _, err := os.Stat(defaultCascadeFilesDir); os.IsNotExist(err) {
+		// directory does not exist, create it
+		errDir := os.MkdirAll(defaultCascadeFilesDir, 0755)
+		if errDir != nil {
+			log.Fatal(err)
+		}
+	}
+
 	// pastelClient reads in the json-formatted hostname, port, username, and password and
 	//  connects over gRPC to cNode for access to Blockchain, Masternodes, Tickets, and PastelID databases
 	pastelClient := pastel.NewClient(config.Pastel, config.Pastel.BurnAddress())
@@ -221,10 +231,13 @@ func runApp(ctx context.Context, config *configs.Config) error {
 	//  Since the API Server has access to the services, this is what finally exposes useful methods like
 	//  "NftGet" and "Download".
 	apiSrcvConf := &services.Config{
-		StaticFilesDir: defaultStaticFilesDir,
+		StaticFilesDir:  defaultStaticFilesDir,
+		CascadeFilesDir: defaultCascadeFilesDir,
 	}
 	config.API.StaticFilesDir = defaultStaticFilesDir
+	config.API.CascadeFilesDir = defaultCascadeFilesDir
 	config.NftDownload.StaticDir = defaultStaticFilesDir
+	config.NftDownload.CascadeFilesDir = defaultCascadeFilesDir
 
 	// These services connect the different clients and configs together to provide tasking and handling for
 	//  the required functionality.  These services aren't started with these declarations, they will be run

walletnode/services/common/7zip.go

@@ -0,0 +1,87 @@
+package common
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/pastelnetwork/gonode/common/errors"
+)
+
+type FileSplitter struct {
+	PartSizeMB int
+}
+
+func get7zPath() string {
+	basePath := "/usr/bin/" // Adjust the base path as needed. For example: ./bin/ for local dir
+	switch runtime.GOOS {
+	case "windows":
+		return basePath + "7za.exe"
+	default:
+		return basePath + "7za"
+	}
+}
+
+func (fs *FileSplitter) SplitFile(filePath string) error {
+	sevenZPath := get7zPath()
+	cmd := exec.Command(sevenZPath, "a", "-v"+strconv.Itoa(fs.PartSizeMB)+"m", filePath+".7z", filePath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return errors.New("7zip split error: " + err.Error())
+	}
+
+	return nil
+}
+
+func (fs *FileSplitter) JoinFiles(dirPath string) error {
+	// Find the first part of the split files
+	var firstPartPath string
+	files, err := os.ReadDir(dirPath)
+	if err != nil {
+		return fmt.Errorf("failed to read directory: %v", err)
+	}
+
+	for _, file := range files {
+		if strings.HasSuffix(file.Name(), ".001") {
+			firstPartPath = filepath.Join(dirPath, file.Name())
+			break
+		}
+	}
+
+	if firstPartPath == "" {
+		return errors.New("no split parts (.001 file) found in the directory")
+	}
+
+	// Join the split parts
+	sevenZPath := get7zPath()
+	fmt.Println("Found .001 file at:", firstPartPath)
+	cmd := exec.Command(sevenZPath, "x", "-aoa", "-o"+dirPath, firstPartPath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Println("Executing join command:", cmd)
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("7zip join error: %v", err)
+	}
+
+	// The assumed path of the joined archive
+	reassembledPath := strings.TrimSuffix(firstPartPath, ".7z.001")
+	fmt.Println("Assumed path of the joined archive:", reassembledPath)
+
+	// Verify the existence of the joined archive
+	if _, err := os.Stat(reassembledPath); os.IsNotExist(err) {
+		fmt.Println("Checking for potential naming error...")
+		if _, err := os.Stat(reassembledPath + ".7z"); err == nil {
+			reassembledPath += ".7z"
+			fmt.Println("Corrected path to joined archive:", reassembledPath)
+		} else {
+			return fmt.Errorf("joined archive does not exist at expected path: %s", reassembledPath)
+		}
+	}
+
+	return nil
+}

walletnode/services/download/config.go

@@ -6,8 +6,9 @@ import (
 
 // Config contains settings of the registering nft.
 type Config struct {
-	common.Config `mapstructure:",squash" json:"-"`
-	StaticDir     string `mapstructure:"static_dir" json:"static_dir"`
+	common.Config   `mapstructure:",squash" json:"-"`
+	StaticDir       string `mapstructure:"static_dir" json:"static_dir"`
+	CascadeFilesDir string `mapstructure:"cascade_files_dir" json:"cascade_files_dir"`
 }
 
 // NewConfig returns a new Config instance.