From e5c3196b31d327c3f14a5f4cd87d91fe77467712 Mon Sep 17 00:00:00 2001 From: Karan Sharma Date: Tue, 15 Oct 2019 17:51:32 +0530 Subject: [PATCH] feat: Add blobstore package This commit introduces a `blobstore` package and refactors the existing upload mechanism. Upload is now handled by `providers` and the two bundled providers are `S3` and `Filesystem`. `app.Blobstore` initialises the correct provider based on the configuration and handles `Put`, `Delete` and `Get` operations. --- .dockerignore | 25 ++++ .gitignore | 1 + admin.go | 1 - config.toml.sample | 38 ++++-- go.mod | 4 + go.sum | 9 ++ main.go | 42 +++++- media.go | 63 +++++---- media/media.go | 26 ++++ media/providers/filesystem/filesystem.go | 133 +++++++++++++++++++ media/providers/s3/s3.go | 108 ++++++++++++++++ models/models.go | 13 -- utils.go | 157 +++++++---------------- 13 files changed, 462 insertions(+), 158 deletions(-) create mode 100644 .dockerignore create mode 100644 media/media.go create mode 100644 media/providers/filesystem/filesystem.go create mode 100644 media/providers/s3/s3.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..3729ff0cd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6a9a8f8dd..a8eba732d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ frontend/yarn.lock config.toml node_modules listmonk +dist/* \ No newline at end of file diff --git a/admin.go b/admin.go index 5200feaed..7a33a1cf6 100644 --- a/admin.go +++ b/admin.go @@ -28,7 +28,6 @@ func handleGetConfigScript(c echo.Context) error { app = c.Get("app").(*App) out = configScript{ RootURL: app.Constants.RootURL, - UploadURI: app.Constants.UploadURI, FromEmail: app.Constants.FromEmail, Messengers: app.Manager.GetMessengerNames(), } diff --git a/config.toml.sample b/config.toml.sample index f58b0b934..829e54255 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -24,14 +24,6 @@ from_email = "listmonk " # To disable notifications, set an empty list, eg: notify_emails = [] notify_emails = ["admin1@mysite.com", "admin2@mysite.com"] -# Path to the uploads directory where media will be uploaded. -upload_path = "uploads" - -# Upload URI that's visible to the outside world. The media -# uploaded to upload_path will be made available publicly -# under this URI, for instance, list.yoursite.com/uploads. -upload_uri = "/uploads" - # Maximum concurrent workers that will attempt to send messages # simultaneously. This should depend on the number of CPUs the # machine has and also the number of simultaenous e-mails the @@ -110,3 +102,33 @@ ssl_mode = "disable" # Maximum concurrent connections to the SMTP server. max_conns = 10 + +# Upload settings +[upload] +# Provider which will be used to host uploaded media. Bundled providers are "filesystem" and "s3". +provider = "filesystem" + +# S3 Provider settings +[upload.s3] +# (Optional). AWS Access Key and Secret Key for the user to access the bucket. Leaving it empty would default to use +# instance IAM role. +aws_access_key_id = "" +aws_secret_access_key = "" +# AWS Region where S3 bucket is hosted. +aws_default_region="ap-south-1" +# Specify bucket name. +bucket="" +# Path where the files will be stored inside bucket. Empty value ("") means the root of bucket. +bucket_path="" +# Bucket type can be "private" or "public". +bucket_type="public" +# (Optional) Specify TTL (in seconds) for the generated presigned URL. Expiry value is used only if the bucket is private. +expiry="86400" + +# Filesystem provider settings +[upload.filesystem] +# Path to the uploads directory where media will be uploaded. Leaving it empty ("") means current working directory. +upload_path="" +# Upload URI that's visible to the outside world. The media uploaded to upload_path will be made available publicly +# under this URI, for instance, list.yoursite.com/uploads. +upload_uri = "/uploads" diff --git a/go.mod b/go.mod index 2ccea4f00..bb829e471 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/knadh/listmonk require ( github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf + github.com/aws/aws-sdk-go v1.25.12 github.com/disintegration/imaging v1.5.0 github.com/jinzhu/gorm v1.9.1 github.com/jmoiron/sqlx v1.2.0 @@ -15,6 +16,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect + github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 github.com/satori/go.uuid v1.2.0 github.com/spf13/pflag v1.0.3 github.com/stretchr/objx v0.2.0 // indirect @@ -27,3 +29,5 @@ require ( gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b ) + +go 1.13 diff --git a/go.sum b/go.sum index 2a023d789..10966391c 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.25.12 h1:a4h2FxoUJq9h+hajSE/dsRiqoOniIh6BkzhxMjkepzY= +github.com/aws/aws-sdk-go v1.25.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -16,6 +18,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA= github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jordan-wright/email v0.0.0-20181027021455-480bedc4908b h1:veTPVnbkOijplSJVywDYKDRPoZEN39kfuMDzzRKP0FA= @@ -57,6 +61,10 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rhnvrm/simples3 v0.2.3 h1:qNXPynabu8M3F4+69fspA5aWZR8jqVV1RQtv2xc1OVk= +github.com/rhnvrm/simples3 v0.2.3/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA= +github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 h1:2josYcx2gm3CT0WMqi0jBagvg50V3UMWlYN/CnBEbSI= +github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= @@ -73,6 +81,7 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM= golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/main.go b/main.go index db507ec15..62e94fbf6 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,9 @@ import ( "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/providers/posflag" "github.com/knadh/listmonk/manager" + "github.com/knadh/listmonk/media" + "github.com/knadh/listmonk/media/providers/filesystem" + "github.com/knadh/listmonk/media/providers/s3" "github.com/knadh/listmonk/messenger" "github.com/knadh/listmonk/subimporter" "github.com/knadh/stuffbin" @@ -30,8 +33,6 @@ type constants struct { RootURL string `koanf:"root"` LogoURL string `koanf:"logo_url"` FaviconURL string `koanf:"favicon_url"` - UploadPath string `koanf:"upload_path"` - UploadURI string `koanf:"upload_uri"` FromEmail string `koanf:"from_email"` NotifyEmails []string `koanf:"notify_emails"` Privacy privacyOptions `koanf:"privacy"` @@ -56,6 +57,7 @@ type App struct { Logger *log.Logger NotifTpls *template.Template Messenger messenger.Messenger + Media media.Store } var ( @@ -195,6 +197,33 @@ func initMessengers(r *manager.Manager) messenger.Messenger { return msgr } +// initMediaStore initializes Upload manager with a custom backend. +func initMediaStore() media.Store { + switch provider := ko.String("upload.provider"); provider { + case "s3": + var opts s3.Opts + ko.Unmarshal("upload.s3", &opts) + uplder, err := s3.NewS3Store(opts) + if err != nil { + logger.Fatalf("error initializing s3 upload provider %s", err) + } + return uplder + case "filesystem": + var opts filesystem.Opts + ko.Unmarshal("upload.filesystem", &opts) + opts.UploadPath = filepath.Clean(opts.UploadPath) + opts.UploadURI = filepath.Clean(opts.UploadURI) + uplder, err := filesystem.NewDiskStore(opts) + if err != nil { + logger.Fatalf("error initializing filesystem upload provider %s", err) + } + return uplder + default: + logger.Fatalf("unknown provider. please select one of either filesystem or s3") + } + return nil +} + func main() { // Connect to the DB. db, err := connectDB(ko.String("db.host"), @@ -216,8 +245,6 @@ func main() { log.Fatalf("error loading app config: %v", err) } c.RootURL = strings.TrimRight(c.RootURL, "/") - c.UploadURI = filepath.Clean(c.UploadURI) - c.UploadPath = filepath.Clean(c.UploadPath) c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable")) // Initialize the static file system into which all @@ -299,6 +326,9 @@ func main() { // Add messengers. app.Messenger = initMessengers(app.Manager) + // Add uploader + app.Media = initMediaStore() + // Initialize the workers that push out messages. go m.Run(time.Second * 5) m.SpawnWorkers() @@ -330,7 +360,9 @@ func main() { fSrv := app.FS.FileServer() srv.GET("/public/*", echo.WrapHandler(fSrv)) srv.GET("/frontend/*", echo.WrapHandler(fSrv)) - srv.Static(c.UploadURI, c.UploadURI) + if ko.String("upload.provider") == "filesystem" { + srv.Static(ko.String("upload.filesystem.upload_uri"), ko.String("upload.filesystem.upload_path")) + } registerHandlers(srv) srv.Logger.Fatal(srv.Start(ko.String("app.address"))) } diff --git a/media.go b/media.go index 2444b93d2..1f3fb9f3f 100644 --- a/media.go +++ b/media.go @@ -3,12 +3,9 @@ package main import ( "fmt" "net/http" - "os" - "path/filepath" "strconv" - "github.com/disintegration/imaging" - "github.com/knadh/listmonk/models" + "github.com/knadh/listmonk/media" "github.com/labstack/echo" uuid "github.com/satori/go.uuid" ) @@ -26,45 +23,64 @@ func handleUploadMedia(c echo.Context) error { app = c.Get("app").(*App) cleanUp = false ) - + file, err := c.FormFile("file") + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, + fmt.Sprintf("Invalid file uploaded: %v", err)) + } + // Validate MIME type with the list of allowed types. + var typ = file.Header.Get("Content-type") + ok := validateMIME(typ, imageMimes) + if !ok { + return echo.NewHTTPError(http.StatusBadRequest, + fmt.Sprintf("Unsupported file type (%s) uploaded.", typ)) + } + // Generate filename + fName := generateFileName(file.Filename) + // Read file contents in memory + src, err := file.Open() + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, + fmt.Sprintf("Error reading file: %s", err)) + } + defer src.Close() // Upload the file. - fName, err := uploadFile("file", app.Constants.UploadPath, "", imageMimes, c) + fName, err = app.Media.Put(fName, typ, src) if err != nil { + cleanUp = true return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error uploading file: %s", err)) } - path := filepath.Join(app.Constants.UploadPath, fName) defer func() { // If any of the subroutines in this function fail, // the uploaded image should be removed. if cleanUp { - os.Remove(path) + app.Media.Delete(fName) + app.Media.Delete(thumbPrefix + fName) } }() - // Create a thumbnail. - src, err := imaging.Open(path) + // Create thumbnail from file. + thumbFile, err := createThumbnail(file) if err != nil { cleanUp = true return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error opening image for resizing: %s", err)) } - - t := imaging.Resize(src, thumbnailSize, 0, imaging.Lanczos) - if err := imaging.Save(t, fmt.Sprintf("%s/%s%s", app.Constants.UploadPath, thumbPrefix, fName)); err != nil { + // Upload thumbnail. + thumbfName, err := app.Media.Put(thumbPrefix+fName, typ, thumbFile) + if err != nil { cleanUp = true return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error saving thumbnail: %s", err)) } - // Write to the DB. - if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, fmt.Sprintf("%s%s", thumbPrefix, fName), 0, 0); err != nil { + if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, thumbfName, 0, 0); err != nil { cleanUp = true return echo.NewHTTPError(http.StatusInternalServerError, - fmt.Sprintf("Error saving uploaded file: %s", pqErrMsg(err))) + fmt.Sprintf("Error saving uploaded file to db: %s", pqErrMsg(err))) } - return c.JSON(http.StatusOK, okResp{true}) } @@ -72,7 +88,7 @@ func handleUploadMedia(c echo.Context) error { func handleGetMedia(c echo.Context) error { var ( app = c.Get("app").(*App) - out []models.Media + out []media.Media ) if err := app.Queries.GetMedia.Select(&out); err != nil { @@ -81,8 +97,8 @@ func handleGetMedia(c echo.Context) error { } for i := 0; i < len(out); i++ { - out[i].URI = fmt.Sprintf("%s/%s", app.Constants.UploadURI, out[i].Filename) - out[i].ThumbURI = fmt.Sprintf("%s/%s%s", app.Constants.UploadURI, thumbPrefix, out[i].Filename) + out[i].URI = app.Media.Get(out[i].Filename) + out[i].ThumbURI = app.Media.Get(thumbPrefix + out[i].Filename) } return c.JSON(http.StatusOK, okResp{out}) @@ -99,13 +115,14 @@ func handleDeleteMedia(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.") } - var m models.Media + var m media.Media if err := app.Queries.DeleteMedia.Get(&m, id); err != nil { return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error deleting media: %s", pqErrMsg(err))) } - os.Remove(filepath.Join(app.Constants.UploadPath, m.Filename)) - os.Remove(filepath.Join(app.Constants.UploadPath, fmt.Sprintf("%s%s", thumbPrefix, m.Filename))) + + app.Media.Delete(m.Filename) + app.Media.Delete(thumbPrefix + m.Filename) return c.JSON(http.StatusOK, okResp{true}) } diff --git a/media/media.go b/media/media.go new file mode 100644 index 000000000..2e89c7c40 --- /dev/null +++ b/media/media.go @@ -0,0 +1,26 @@ +package media + +import ( + "io" + + "gopkg.in/volatiletech/null.v6" +) + +// Media represents an uploaded object. +type Media struct { + ID int `db:"id" json:"id"` + UUID string `db:"uuid" json:"uuid"` + Filename string `db:"filename" json:"filename"` + Width int `db:"width" json:"width"` + Height int `db:"height" json:"height"` + CreatedAt null.Time `db:"created_at" json:"created_at"` + ThumbURI string `json:"thumb_uri"` + URI string `json:"uri"` +} + +// Store represents set of methods to perform upload/delete operations. +type Store interface { + Put(string, string, io.ReadSeeker) (string, error) + Delete(string) error + Get(string) string +} diff --git a/media/providers/filesystem/filesystem.go b/media/providers/filesystem/filesystem.go new file mode 100644 index 000000000..4d9c19a67 --- /dev/null +++ b/media/providers/filesystem/filesystem.go @@ -0,0 +1,133 @@ +package filesystem + +import ( + "crypto/rand" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/knadh/listmonk/media" +) + +const tmpFilePrefix = "listmonk" + +// Opts represents filesystem params +type Opts struct { + UploadPath string `koanf:"upload_path"` + UploadURI string `koanf:"upload_uri"` +} + +// Client implements `media.Store` +type Client struct { + opts Opts +} + +// NewDiskStore initialises store for Filesystem provider. +func NewDiskStore(opts Opts) (media.Store, error) { + return &Client{ + opts: opts, + }, nil +} + +// Put accepts the filename, the content type and file object itself and stores the file in disk. +func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) { + var out *os.File + // There's no explicit name. Use the one posted in the HTTP request. + if filename == "" { + filename = strings.TrimSpace(filename) + if filename == "" { + filename, _ = generateRandomString(10) + } + } + // Get the directory path + dir := getDir(e.opts.UploadPath) + filename = assertUniqueFilename(dir, filename) + o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) + if err != nil { + return "", err + } + out = o + defer out.Close() + + if _, err := io.Copy(out, src); err != nil { + return "", err + } + return filename, nil +} + +// Get accepts a filename and retrieves the full path from disk. +func (e *Client) Get(name string) string { + return fmt.Sprintf("%s/%s", e.opts.UploadURI, name) +} + +// Delete accepts a filename and removes it from disk. +func (e *Client) Delete(file string) error { + dir := getDir(e.opts.UploadPath) + err := os.Remove(filepath.Join(dir, file)) + if err != nil { + return err + } + return nil +} + +// This matches filenames, sans extensions, of the format +// filename_(number). The number is incremented in case +// new file uploads conflict with existing filenames +// on the filesystem. +var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`) + +// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't, +// it returns the same name and if it does, it adds a small random hash to the filename +// and returns that. +func assertUniqueFilename(dir, fileName string) string { + var ( + ext = filepath.Ext(fileName) + base = fileName[0 : len(fileName)-len(ext)] + num = 0 + ) + + for { + // There's no name conflict. + if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) { + return fileName + } + + // Does the name match the _(num) syntax? + r := fnameRegexp.FindAllStringSubmatch(fileName, -1) + if len(r) == 1 && len(r[0]) == 3 { + num, _ = strconv.Atoi(r[0][2]) + } + num++ + + fileName = fmt.Sprintf("%s_%d%s", base, num, ext) + } +} + +// generateRandomString generates a cryptographically random, alphanumeric string of length n. +func generateRandomString(n int) (string, error) { + const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + var bytes = make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + + for k, v := range bytes { + bytes[k] = dictionary[v%byte(len(dictionary))] + } + + return string(bytes), nil +} + +// getDir returns the current working directory path if no directory is specified, +// else returns the directory path specified itself. +func getDir(dir string) string { + if dir == "" { + dir, _ = os.Getwd() + } + return dir +} diff --git a/media/providers/s3/s3.go b/media/providers/s3/s3.go new file mode 100644 index 000000000..653db81b8 --- /dev/null +++ b/media/providers/s3/s3.go @@ -0,0 +1,108 @@ +package s3 + +import ( + "errors" + "fmt" + "io" + "time" + + "github.com/knadh/listmonk/media" + "github.com/rhnvrm/simples3" +) + +const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s" + +// Opts represents AWS S3 specific params +type Opts struct { + AccessKey string `koanf:"aws_access_key_id"` + SecretKey string `koanf:"aws_secret_access_key"` + Region string `koanf:"aws_default_region"` + Bucket string `koanf:"bucket"` + BucketPath string `koanf:"bucket_path"` + BucketType string `koanf:"bucket_type"` + Expiry int `koanf:"expiry"` +} + +// Client implements `media.Store` for S3 provider +type Client struct { + s3 *simples3.S3 + opts Opts +} + +// NewS3Store initialises store for S3 provider. It takes in the AWS configuration +// and sets up the `simples3` client to interact with AWS APIs for all bucket operations. +func NewS3Store(opts Opts) (media.Store, error) { + var s3svc *simples3.S3 + var err error + if opts.Region == "" { + return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config") + } + // Use Access Key/Secret Key if specified in config. + if opts.AccessKey != "" && opts.SecretKey != "" { + s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey) + } else { + // fallback to IAM role if no access key/secret key is provided. + s3svc, err = simples3.NewUsingIAM(opts.Region) + if err != nil { + return nil, err + } + } + return &Client{ + s3: s3svc, + opts: opts, + }, nil +} + +// Put takes in the filename, the content type and file object itself and uploads to S3. +func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) { + // Upload input parameters + upParams := simples3.UploadInput{ + Bucket: e.opts.Bucket, + ObjectKey: getBucketPath(e.opts.BucketPath, name), + ContentType: cType, + FileName: name, + Body: file, + } + // Perform an upload. + _, err := e.s3.FileUpload(upParams) + if err != nil { + return "", err + } + return name, nil +} + +// Get accepts the filename of the object stored and retrieves from S3. +func (e *Client) Get(name string) string { + // Generate a private S3 pre-signed URL if it's a private bucket. + if e.opts.BucketType == "private" { + url := e.s3.GeneratePresignedURL(simples3.PresignedInput{ + Bucket: e.opts.Bucket, + ObjectKey: getBucketPath(e.opts.BucketPath, name), + Method: "GET", + Timestamp: time.Now(), + ExpirySeconds: e.opts.Expiry, + }) + return url + } + // Generate a public S3 URL if it's a public bucket. + url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name)) + return url +} + +// Delete accepts the filename of the object and deletes from S3. +func (e *Client) Delete(name string) error { + err := e.s3.FileDelete(simples3.DeleteInput{ + Bucket: e.opts.Bucket, + ObjectKey: getBucketPath(e.opts.BucketPath, name), + }) + return err +} + +// getBucketPath constructs the key for the object stored in S3. +// If path is empty, the key is the combination of root of S3 bucket and filename. +func getBucketPath(path string, name string) string { + if path == "" { + return fmt.Sprintf("%s", name) + } + return fmt.Sprintf("%s/%s", path, name) +} diff --git a/models/models.go b/models/models.go index 4efaaef21..60c77e278 100644 --- a/models/models.go +++ b/models/models.go @@ -174,19 +174,6 @@ type CampaignMeta struct { // Campaigns represents a slice of Campaigns. type Campaigns []Campaign -// Media represents an uploaded media item. -type Media struct { - ID int `db:"id" json:"id"` - UUID string `db:"uuid" json:"uuid"` - Filename string `db:"filename" json:"filename"` - Width int `db:"width" json:"width"` - Height int `db:"height" json:"height"` - CreatedAt null.Time `db:"created_at" json:"created_at"` - - ThumbURI string `json:"thumb_uri"` - URI string `json:"uri"` -} - // Template represents a reusable e-mail template. type Template struct { Base diff --git a/utils.go b/utils.go index bb1cc52bc..40f2ffc14 100644 --- a/utils.go +++ b/utils.go @@ -4,30 +4,22 @@ import ( "bytes" "crypto/rand" "fmt" - "io" - "io/ioutil" + "log" + "mime/multipart" "net/http" - "os" - "path/filepath" "reflect" "regexp" "strconv" "strings" + "github.com/disintegration/imaging" "github.com/jmoiron/sqlx" "github.com/knadh/goyesql" "github.com/labstack/echo" "github.com/lib/pq" ) -const tmpFilePrefix = "listmonk" - var ( - // This matches filenames, sans extensions, of the format - // filename_(number). The number is incremented in case - // new file uploads conflict with existing filenames - // on the filesystem. - fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`) // This replaces all special characters tagRegexp = regexp.MustCompile(`[^a-z0-9\-\s]`) @@ -95,23 +87,12 @@ func scanQueriesToStruct(obj interface{}, q goyesql.Queries, db *sqlx.DB) error return nil } -// uploadFile is a helper function on top of echo.Context for processing file uploads. -// It allows copying a single file given the incoming file field name. -// If the upload directory dir is empty, the file is copied to the system's temp directory. -// If name is empty, the incoming file's name along with a small random hash is used. -// When a slice of MIME types is given, the uploaded file's MIME type is validated against the list. -func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (string, error) { - file, err := c.FormFile(key) - if err != nil { - return "", echo.NewHTTPError(http.StatusBadRequest, - fmt.Sprintf("Invalid file uploaded: %v", err)) - } - - // Check MIME type. +// validateMIME is a helper function to validate uploaded file's MIME type +// against the slice of MIME types is given. +func validateMIME(typ string, mimes []string) (ok bool) { if len(mimes) > 0 { var ( - typ = file.Header.Get("Content-type") - ok = false + ok = false ) for _, m := range mimes { if typ == m { @@ -119,55 +100,42 @@ func uploadFile(key string, dir, name string, mimes []string, c echo.Context) (s break } } - if !ok { - return "", echo.NewHTTPError(http.StatusBadRequest, - fmt.Sprintf("Unsupported file type (%s) uploaded.", typ)) + return false } } + return true +} + +// generateFileName appends the incoming file's name with a small random hash. +func generateFileName(fName string) string { + name := strings.TrimSpace(fName) + if name == "" { + name, _ = generateRandomString(10) + } + return name +} +// createThumbnail reads the file object and returns a smaller image +func createThumbnail(file *multipart.FileHeader) (*bytes.Reader, error) { src, err := file.Open() if err != nil { - return "", err + return nil, err } defer src.Close() - - // There's no upload directory. Use a tempfile. - var out *os.File - if dir == "" { - o, err := ioutil.TempFile("", tmpFilePrefix) - if err != nil { - return "", echo.NewHTTPError(http.StatusInternalServerError, - fmt.Sprintf("Error copying uploaded file: %v", err)) - } - out = o - name = o.Name() - } else { - // There's no explicit name. Use the one posted in the HTTP request. - if name == "" { - name = strings.TrimSpace(file.Filename) - if name == "" { - name, _ = generateRandomString(10) - } - } - name = assertUniqueFilename(dir, name) - - o, err := os.OpenFile(filepath.Join(dir, name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) - if err != nil { - return "", echo.NewHTTPError(http.StatusInternalServerError, - fmt.Sprintf("Error copying uploaded file: %v", err)) - } - - out = o + img, err := imaging.Decode(src) + if err != nil { + return nil, echo.NewHTTPError(http.StatusInternalServerError, + fmt.Sprintf("Error decoding image: %v", err)) } - defer out.Close() - - if _, err = io.Copy(out, src); err != nil { - return "", echo.NewHTTPError(http.StatusInternalServerError, - fmt.Sprintf("Error copying uploaded file: %v", err)) + t := imaging.Resize(img, thumbnailSize, 0, imaging.Lanczos) + // Encode the image into a byte slice as PNG. + var buf bytes.Buffer + err = imaging.Encode(&buf, t, imaging.PNG) + if err != nil { + log.Fatal(err) } - - return name, nil + return bytes.NewReader(buf.Bytes()), nil } // Given an error, pqErrMsg will try to return pq error details @@ -182,49 +150,6 @@ func pqErrMsg(err error) string { return err.Error() } -// generateRandomString generates a cryptographically random, alphanumeric string of length n. -func generateRandomString(n int) (string, error) { - const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - - var bytes = make([]byte, n) - if _, err := rand.Read(bytes); err != nil { - return "", err - } - - for k, v := range bytes { - bytes[k] = dictionary[v%byte(len(dictionary))] - } - - return string(bytes), nil -} - -// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't, -// it returns the same name and if it does, it adds a small random hash to the filename -// and returns that. -func assertUniqueFilename(dir, fileName string) string { - var ( - ext = filepath.Ext(fileName) - base = fileName[0 : len(fileName)-len(ext)] - num = 0 - ) - - for { - // There's no name conflict. - if _, err := os.Stat(filepath.Join(dir, fileName)); os.IsNotExist(err) { - return fileName - } - - // Does the name match the _(num) syntax? - r := fnameRegexp.FindAllStringSubmatch(fileName, -1) - if len(r) == 1 && len(r[0]) == 3 { - num, _ = strconv.Atoi(r[0][2]) - } - num++ - - fileName = fmt.Sprintf("%s_%d%s", base, num, ext) - } -} - // normalizeTags takes a list of string tags and normalizes them by // lowercasing and removing all special characters except for dashes. func normalizeTags(tags []string) []string { @@ -282,3 +207,19 @@ func parseStringIDs(s []string) ([]int64, error) { return vals, nil } + +// generateRandomString generates a cryptographically random, alphanumeric string of length n. +func generateRandomString(n int) (string, error) { + const dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + var bytes = make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + + for k, v := range bytes { + bytes[k] = dictionary[v%byte(len(dictionary))] + } + + return string(bytes), nil +}