From 9a4bbac82944c06c783ef0e2f300dda9f13333ee Mon Sep 17 00:00:00 2001 From: Miguel Ramos Date: Thu, 11 Feb 2021 19:27:15 +0000 Subject: [PATCH] feat: upload resource simple --- api/api.go | 2 +- api/bucket.go | 78 ++++++++++++++++++++++++++++++++-- api/middleware.go | 25 ++++++++++- api/minio.go | 16 ++++++- config/minio.go | 2 +- config/types.go | 21 +++------ database/00-gotrue-schema.sql | 13 ++++++ database/01-storage-schema.sql | 9 +++- go.mod | 1 + go.sum | 2 + models/bucket.go | 19 +++++---- models/bucket_media.go | 38 +++++++++++++++++ models/identity.go | 65 ++++++++++++++++++++++++++++ models/media.go | 57 +++++++++++++++++++++++++ storage/dial.go | 1 + utils/helpers.go | 6 +++ 16 files changed, 321 insertions(+), 34 deletions(-) create mode 100644 models/bucket_media.go create mode 100644 models/identity.go create mode 100644 models/media.go diff --git a/api/api.go b/api/api.go index 44b6b1c..282ab22 100644 --- a/api/api.go +++ b/api/api.go @@ -42,7 +42,7 @@ func NewPrivateAPI(api *API, router fiber.Router) { bucketRouter := router.Group("/bucket") bucketRouter.Post("", api.CreateBucket) - bucketRouter.Post("/upload", api.PutObject) + bucketRouter.Post("/:bucket", api.PutObject) userRouter := router.Group("/user") userRouter.Post("", api.CreateUser) diff --git a/api/bucket.go b/api/bucket.go index 10fc4d9..7af955a 100644 --- a/api/bucket.go +++ b/api/bucket.go @@ -1,11 +1,17 @@ package api import ( + "encoding/json" "fmt" + "net/url" + "os" + "path" "strings" + "github.com/barasher/go-exiftool" "github.com/gobuffalo/pop/nulls" "github.com/gofiber/fiber/v2" + "github.com/gofrs/uuid" "github.com/gosimple/slug" "github.com/minio/minio-go/v7" "github.com/websublime/barrel/config" @@ -79,8 +85,74 @@ func (api *API) CreateBucket(ctx *fiber.Ctx) error { }) } +// PutObject upload to bucket func (api *API) PutObject(ctx *fiber.Ctx) error { - return ctx.JSON(fiber.Map{ - "data": "object", - }) + bucketName := ctx.Params("bucket") + medias := []*models.Media{} + //TODO: user + bucket, err := models.FindBucket(api.db, bucketName) + if err != nil { + return utils.NewException(utils.ErrorBucketMissing, fiber.StatusBadRequest, err.Error()) + } + + if et, err := exiftool.NewExiftool(); err == nil { + defer et.Close() + + if form, err := ctx.MultipartForm(); err == nil { + files := form.File["asset"] + + for _, file := range files { + ctx.SaveFile(file, fmt.Sprintf("./temp/%s", file.Filename)) + data := et.ExtractMetadata(fmt.Sprintf("./temp/%s", file.Filename)) + + meta, err := json.Marshal(data[0].Fields) + if err != nil { + return utils.NewException(utils.ErrorResourceMetaFailure, fiber.StatusBadRequest, err.Error()) + } + + metafile := new(config.MetaFile) + json.Unmarshal([]byte(meta), &metafile) + + bucketFile, err := api.store.FPutObject(ctx.Context(), bucketName, metafile.FileName, fmt.Sprintf("./temp/%s", metafile.FileName), minio.PutObjectOptions{ + ContentType: metafile.MIMEType, + UserMetadata: map[string]string{}, + }) + + if err != nil { + return utils.NewException(utils.ErrorResourceBucketFailure, fiber.StatusBadRequest, err.Error()) + } + + u, _ := url.Parse(api.config.BarrelBaseURL) + u.Path = path.Join(u.Path, bucketName, metafile.FileName) + bucketFileJSON, _ := json.Marshal(bucketFile) + metaFileJSON, _ := json.Marshal(metafile) + + media, _ := models.NewMedia(u.String(), nulls.NewUUID(uuid.Nil), nulls.NewString(string(bucketFileJSON)), nulls.NewString(string(metaFileJSON))) + bucketMedia, _ := models.NewBucketMedia(bucket.ID, media.ID) + + err = api.db.Transaction(func(tx *storage.Connection) error { + terr := tx.Create(media) + terr = tx.Create(bucketMedia) + + return terr + }) + if err != nil { + return utils.NewException(utils.ErrorResourceModelSave, fiber.StatusBadRequest, err.Error()) + } + + os.Remove(fmt.Sprintf("./temp/%s", file.Filename)) + + medias = append(medias, media) + } + + return ctx.JSON(fiber.Map{ + "data": medias, + }) + } else { + return utils.NewException(utils.ErrorResourceInvalidForm, fiber.StatusBadRequest, err.Error()) + } + } else { + return utils.NewException(utils.ErrorExifMissing, fiber.StatusBadRequest, err.Error()) + } + } diff --git a/api/middleware.go b/api/middleware.go index 98a5412..1367cb9 100644 --- a/api/middleware.go +++ b/api/middleware.go @@ -9,6 +9,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/websublime/barrel/config" + "github.com/websublime/barrel/models" "github.com/websublime/barrel/utils" ) @@ -42,7 +43,7 @@ func (api *API) AuthorizedMiddleware(ctx *fiber.Ctx) error { } ctx.Locals("claims", claims) - ctx.Locals("token", auth) + ctx.Locals("token", token) } else { return utils.NewException(utils.ErrorOrgStatusForbidden, fiber.StatusForbidden, "Only authorized requests are permitted") } @@ -57,7 +58,14 @@ func (api *API) AdminMiddleware(ctx *fiber.Ctx) error { headerKey := ctx.Get("X-BARREL-KEY") - if len(headerKey) > 0 && strings.Compare(headerKey, api.config.BarrelAdminKey) == 0 { + identity, err := models.FindIdentityByKey(api.db, headerKey) + if err != nil { + return utils.NewException(utils.ErrorOrgStatusForbidden, fiber.StatusForbidden, err.Error()) + } + + if identity.IsAdmin { + isAdmin = true + } else if len(headerKey) > 0 && strings.Compare(headerKey, api.config.BarrelAdminKey) == 0 { isAdmin = true } else { claimer := ctx.Locals("claims").(*config.GoTrueClaims) @@ -85,14 +93,27 @@ func (api *API) AdminMiddleware(ctx *fiber.Ctx) error { //CanAccessMiddleware check if user is register to access private endpoints func (api *API) CanAccessMiddleware(ctx *fiber.Ctx) error { isAdmin := ctx.Locals("admin").(bool) + token := ctx.Locals("token").(*jwt.Token) headerKey := ctx.Get("X-BARREL-KEY") + identity, err := models.FindIdentityByKey(api.db, headerKey) + if err != nil { + return utils.NewException(utils.ErrorOrgStatusForbidden, fiber.StatusForbidden, err.Error()) + } + if !isAdmin { user, err := config.UserIsRegister(api.config, headerKey) if err != nil { return err } + if user.Status == "disabled" { + return utils.NewException(utils.ErrorOrgStatusForbidden, fiber.StatusForbidden, "User is disabled") + } + // TODO: we need to save user key/secret on database + client, _ := config.NewClient(api.config, identity.AccessKey, identity.SecretKey, token.Raw) + api.store = client + ctx.Locals("user", user) } diff --git a/api/minio.go b/api/minio.go index a4b09b0..6be9ae9 100644 --- a/api/minio.go +++ b/api/minio.go @@ -3,6 +3,8 @@ package api import ( "github.com/gofiber/fiber/v2" "github.com/websublime/barrel/config" + "github.com/websublime/barrel/models" + "github.com/websublime/barrel/storage" "github.com/websublime/barrel/utils" ) @@ -16,7 +18,10 @@ func (api *API) CreateUser(ctx *fiber.Ctx) error { return utils.NewException(utils.ErrorUserCreation, fiber.StatusForbidden, "Creation permission denied") } - identity := new(config.Identity) + identity, err := models.NewIdentity("", "", isAdmin) + if err != nil { + return utils.NewException(utils.ErrorOrgUserFailure, fiber.StatusBadRequest, err.Error()) + } if err := ctx.BodyParser(identity); err != nil { return utils.NewException(utils.ErrorUserBodyParse, fiber.StatusPreconditionFailed, "Invalid request body parser") @@ -30,6 +35,15 @@ func (api *API) CreateUser(ctx *fiber.Ctx) error { return utils.NewException(utils.ErrorOrgUserFailure, fiber.StatusBadRequest, err.Error()) } + err = api.db.Transaction(func(tx *storage.Connection) error { + terr := tx.Create(identity) + + return terr + }) + if err != nil { + return utils.NewException(utils.ErrorResourceModelSave, fiber.StatusBadRequest, err.Error()) + } + return ctx.JSON(fiber.Map{ "data": identity, }) diff --git a/config/minio.go b/config/minio.go index 4f34d5d..6f86209 100644 --- a/config/minio.go +++ b/config/minio.go @@ -39,7 +39,7 @@ func OpenAdminClient(conf *EnvironmentConfig) (*madmin.AdminClient, error) { // NewClient create a new client connection func NewClient(conf *EnvironmentConfig, key string, secret string, token string) (*minio.Client, error) { minioClient, err := minio.New(conf.BarrelMinioURL, &minio.Options{ - Creds: credentials.NewStaticV4(key, secret, token), + Creds: credentials.NewStaticV4(key, secret, ""), Secure: false, }) diff --git a/config/types.go b/config/types.go index a0b5e15..c3fef27 100644 --- a/config/types.go +++ b/config/types.go @@ -14,23 +14,12 @@ type GoTrueClaims struct { UserMetaData map[string]interface{} `json:"user_metadata"` } -type Identity struct { - AccessKey string `json:"accessKey,omitempty"` - SecretKey string `json:"secretKey,omitempty"` -} - -func (identity *Identity) Validate() *validate.Errors { - return validate.Validate( - &validators.StringIsPresent{Field: identity.AccessKey, Name: "AccessKey", Message: "Identity accessKey is missing"}, - &validators.StringIsPresent{Field: identity.SecretKey, Name: "SecretKey", Message: "Identity secretKey is missing"}, - ) -} - type GoTrueIdentity struct { - Identity - ID uuid.UUID `json:"id,omitempty"` - UserID uuid.UUID `json:"userID,omitempty"` - Token string `json:"token,omitempty"` + AccessKey string `json:"accessKey,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + ID uuid.UUID `json:"id,omitempty"` + UserID uuid.UUID `json:"userID,omitempty"` + Token string `json:"token,omitempty"` } type CannedPolicy struct { diff --git a/database/00-gotrue-schema.sql b/database/00-gotrue-schema.sql index 4bbb2a4..781f213 100644 --- a/database/00-gotrue-schema.sql +++ b/database/00-gotrue-schema.sql @@ -84,6 +84,18 @@ CREATE TABLE auth.identities ( user_id uuid NOT NULL, CONSTRAINT identities_pkey PRIMARY KEY (id) ); +-- auth.templates definition +CREATE TABLE auth.templates +( + id uuid NOT NULL, + aud varchar(255), + type varchar(50), + subject text, + url text DEFAULT '/', + base_url varchar(255), + url_template text, + CONSTRAINT templates_pkey PRIMARY KEY (id) +); -- Gets the User ID from the request cookie create or replace function auth.uid() returns uuid as $$ select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; @@ -92,6 +104,7 @@ $$ language sql stable; create or replace function auth.role() returns text as $$ select nullif(current_setting('request.jwt.claim.role', true), '')::text; $$ language sql stable; + GRANT ALL PRIVILEGES ON SCHEMA auth TO postgres; GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA auth TO postgres; GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA auth TO postgres; diff --git a/database/01-storage-schema.sql b/database/01-storage-schema.sql index 08135a6..ebe5af0 100644 --- a/database/01-storage-schema.sql +++ b/database/01-storage-schema.sql @@ -35,5 +35,12 @@ CREATE TABLE "barrel".bucket_media ( ); ALTER TABLE "barrel".bucket_media ADD CONSTRAINT fk_bucket_media_bucket FOREIGN KEY ( bucket_id ) REFERENCES "barrel".buckets( id ); - ALTER TABLE "barrel".bucket_media ADD CONSTRAINT fk_bucket_media_media FOREIGN KEY ( media_id ) REFERENCES "barrel".medias( id ); + +CREATE TABLE "barrel".identities ( + id uuid NOT NULL , + "secret" text NOT NULL , + "key" text NOT NULL , + is_admin boolean DEFAULT false , + CONSTRAINT pk_users_id PRIMARY KEY (id) +); \ No newline at end of file diff --git a/go.mod b/go.mod index 92d750d..78990ab 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/websublime/barrel go 1.15 require ( + github.com/barasher/go-exiftool v1.3.2 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/gobuffalo/pop v4.13.1+incompatible github.com/gobuffalo/pop/v5 v5.3.3 diff --git a/go.sum b/go.sum index 19192af..3f73549 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/barasher/go-exiftool v1.3.2 h1:yWUIGOsM6PLbbHxe84ASTo/eyORMTyMH/5Qv1yBcC7s= +github.com/barasher/go-exiftool v1.3.2/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck= github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= diff --git a/models/bucket.go b/models/bucket.go index 8cb2440..9385ecf 100644 --- a/models/bucket.go +++ b/models/bucket.go @@ -16,15 +16,16 @@ import ( // Bucket model type type Bucket struct { - ID uuid.UUID `json:"id" db:"id"` - Name string `json:"name" db:"name"` - Bucket nulls.String `json:"bucket,omitempty" db:"bucket"` - OrgID nulls.String `json:"orgId,omitempty" db:"org_id"` - IsPrivate bool `json:"isPrivate" db:"is_private"` - Policy string `json:"policy" db:"-"` - CreatedAt time.Time `json:"createdAt" db:"created_at"` - UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` - DeletedAt nulls.Time `json:"deleteAt,omitempty" db:"deleted_at"` + ID uuid.UUID `json:"id" db:"id"` + Name string `json:"name" db:"name"` + Bucket nulls.String `json:"bucket,omitempty" db:"bucket"` + OrgID nulls.String `json:"orgId,omitempty" db:"org_id"` + IsPrivate bool `json:"isPrivate" db:"is_private"` + Policy string `json:"policy" db:"-"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` + DeletedAt nulls.Time `json:"deleteAt,omitempty" db:"deleted_at"` + BucketMedia []*BucketMedia `json:"medias,omitempty" many_to_many:"bucket_media" db:"-" fk_id:"bucket_id"` } // NewBucket creates new Bucket diff --git a/models/bucket_media.go b/models/bucket_media.go new file mode 100644 index 0000000..64b35ba --- /dev/null +++ b/models/bucket_media.go @@ -0,0 +1,38 @@ +package models + +import ( + "github.com/gobuffalo/uuid" + "github.com/pkg/errors" + "github.com/websublime/barrel/storage/namespace" +) + +type BucketMedia struct { + ID uuid.UUID `json:"id" db:"id" primary_id:"id"` + BucketID uuid.UUID `json:"bucketId" db:"bucket_id"` + MediaID uuid.UUID `json:"mediaId" db:"media_id"` +} + +func (BucketMedia) TableName() string { + tableName := "bucket_media" + + if namespace.GetNamespace() != "" { + return namespace.GetNamespace() + "." + tableName + } + + return tableName +} + +func NewBucketMedia(bucket uuid.UUID, media uuid.UUID) (*BucketMedia, error) { + uid, err := uuid.NewV4() + if err != nil { + return nil, errors.Wrap(err, "Error generating unique id") + } + + bucketMedia := &BucketMedia{ + ID: uid, + BucketID: bucket, + MediaID: media, + } + + return bucketMedia, nil +} diff --git a/models/identity.go b/models/identity.go new file mode 100644 index 0000000..192c73e --- /dev/null +++ b/models/identity.go @@ -0,0 +1,65 @@ +package models + +import ( + "database/sql" + + "github.com/gobuffalo/uuid" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/pkg/errors" + "github.com/websublime/barrel/storage" + "github.com/websublime/barrel/storage/namespace" +) + +type Identity struct { + ID uuid.UUID `json:"id" db:"id"` + AccessKey string `json:"accessKey" db:"key"` + SecretKey string `json:"secretKey" db:"secret"` + IsAdmin bool `json:"isAdmin" db:"is_admin"` +} + +func (Identity) TableName() string { + tableName := "identities" + + if namespace.GetNamespace() != "" { + return namespace.GetNamespace() + "." + tableName + } + + return tableName +} + +func NewIdentity(secret string, key string, isAdmin bool) (*Identity, error) { + uid, err := uuid.NewV4() + if err != nil { + return nil, errors.Wrap(err, "Error generating unique id") + } + + user := &Identity{ + ID: uid, + SecretKey: secret, + AccessKey: key, + IsAdmin: isAdmin, + } + + return user, nil +} + +func (u *Identity) Validate() *validate.Errors { + return validate.Validate( + &validators.StringIsPresent{Field: u.AccessKey, Name: "AccessKey", Message: "Secret is missing"}, + &validators.StringIsPresent{Field: u.SecretKey, Name: "SecretKey", Message: "Key is missing"}, + ) +} + +func FindIdentityByKey(tx *storage.Connection, key string) (*Identity, error) { + identity := &Identity{} + if err := tx.Where("key = ?", key).First(identity); err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, errors.Wrap(err, "Identity not found") + } + + return nil, errors.Wrap(err, err.Error()) + } + + return identity, nil +} diff --git a/models/media.go b/models/media.go new file mode 100644 index 0000000..e14c285 --- /dev/null +++ b/models/media.go @@ -0,0 +1,57 @@ +package models + +import ( + "time" + + "github.com/gobuffalo/pop/nulls" + "github.com/gobuffalo/uuid" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/pkg/errors" + "github.com/websublime/barrel/storage/namespace" +) + +type Media struct { + ID uuid.UUID `json:"id" db:"id"` + URL string `json:"url" db:"url"` + Owner nulls.UUID `json:"ownerId" db:"owner"` + BucketFile nulls.String `json:"bucketFile" db:"bucket_file"` + Metafile nulls.String `json:"metadata" db:"meta_file"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` + DeletedAt nulls.Time `json:"deletedAt" db:"deleted_at"` + BucketMedia []*BucketMedia `json:"medias,omitempty" many_to_many:"bucket_media" db:"-" fk_id:"media_id" primary_id:"id"` +} + +func (Media) TableName() string { + tableName := "medias" + + if namespace.GetNamespace() != "" { + return namespace.GetNamespace() + "." + tableName + } + + return tableName +} + +func NewMedia(url string, owner nulls.UUID, bucketFile nulls.String, metadata nulls.String) (*Media, error) { + uid, err := uuid.NewV4() + if err != nil { + return nil, errors.Wrap(err, "Error generating unique id") + } + + media := &Media{ + ID: uid, + URL: url, + Owner: owner, + BucketFile: bucketFile, + Metafile: metadata, + } + + return media, nil +} + +func (m *Media) Validate() *validate.Errors { + return validate.Validate( + &validators.StringIsPresent{Field: m.URL, Name: "URL", Message: "Url is missign"}, + ) +} diff --git a/storage/dial.go b/storage/dial.go index 42aa1e9..2ebb2c6 100644 --- a/storage/dial.go +++ b/storage/dial.go @@ -39,6 +39,7 @@ func Dial(conf *config.EnvironmentConfig) (*Connection, error) { if logrus.StandardLogger().Level == logrus.DebugLevel { pop.Debug = true } + pop.Debug = true return &Connection{db}, nil } diff --git a/utils/helpers.go b/utils/helpers.go index da36a6d..32a621c 100644 --- a/utils/helpers.go +++ b/utils/helpers.go @@ -5,6 +5,7 @@ const ( ErrorUserCreation = "EUSER_CREATEFORBIDDEN" ErrorUserBodyParse = "EUSER_BODYPARSE" ErrorBucketModel = "EBUCKET_MODEL" + ErrorBucketMissing = "EBUCKET_MISSING" ErrorBucketBodyParse = "EBUCKET_BODYPARSE" ErrorBucketExist = "EBUCKET_EXIST" ErrorBucketCreation = "EBUCKET_CREATE" @@ -23,6 +24,11 @@ const ( ErrorParseJson = "EJSON_PARSE" ErrorResourceForbidden = "ERESOURCE_FORBIDDEN" ErrorResourceInvalidBody = "ERESOURCE_BODYINVALID" + ErrorResourceInvalidForm = "ERESOURCE_FORMINVALID" + ErrorResourceMetaFailure = "ERESOURCE_METAFAILURE" + ErrorResourceBucketFailure = "ERESOURCE_BUCKETFAILURE" + ErrorResourceModelSave = "ERESOURCE_MODELSAVE" + ErrorExifMissing = "EEXIF_MISSING" ) // Contains checks if a string is present in a slice