feat: read csv files using csv tags (#152)

Co-authored-by: mj <akbarian.dev@gmail.ccom>

Author: akbariandev

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.27
 	github.com/bwmarrin/discordgo v0.28.1
 	github.com/go-mail/mail/v2 v2.3.0
+	github.com/h2non/gock v1.2.0
+	github.com/jszwec/csvutil v1.10.0
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/pactus-project/pactus v1.3.0
 	github.com/spf13/cobra v1.8.1
@@ -16,6 +18,8 @@ require (
 	gorm.io/gorm v1.25.10
 )
 
+require github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
+
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/NathanBaulch/protoc-gen-cobra v1.2.1 // indirect

go.sum

@@ -76,6 +76,10 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
+github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE=
+github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -120,6 +124,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
+github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
 github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4=
 github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
@@ -212,6 +218,8 @@ github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf
 github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA=
 github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
 github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
 github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
 github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=

internal/engine/command/voucher/create.go

@@ -6,10 +6,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
-	"strings"
 
+	"github.com/jszwec/csvutil"
 	"github.com/pactus-project/pactus/util/logger"
 	"github.com/pagu-project/Pagu/internal/engine/command"
 	"github.com/pagu-project/Pagu/internal/entity"
@@ -19,6 +20,14 @@ import (
 	"gorm.io/datatypes"
 )
 
+type BulkRecorder struct {
+	Recipient        string  `csv:"Recipient"`
+	Email            string  `csv:"Email"`
+	Amount           float64 `csv:"Amount"`
+	ValidatedInMonth int     `csv:"Validated"`
+	Description      string  `csv:"Description"`
+}
+
 func (v *Voucher) createOneHandler(
 	caller *entity.User,
 	cmd *command.Command,
@@ -87,14 +96,27 @@ func (v *Voucher) createBulkHandler(
 		_ = resp.Body.Close()
 	}()
 
-	r := csv.NewReader(resp.Body)
-	records, err := r.ReadAll()
+	csvReader := csv.NewReader(resp.Body)
+	dec, err := csvutil.NewDecoder(csvReader)
 	if err != nil {
 		logger.Error(err.Error())
-		return cmd.ErrorResult(errors.New("failed to read attachment content"))
+		return cmd.ErrorResult(errors.New("failed to read csv content"))
+	}
+
+	var records []BulkRecorder
+	for {
+		r := BulkRecorder{}
+		if err = dec.Decode(&r); errors.Is(err, io.EOF) {
+			break
+		} else if err != nil {
+			logger.Error(err.Error())
+			return cmd.ErrorResult(errors.New("failed to parse csv content"))
+		}
+
+		records = append(records, r)
 	}
 
-	if len(records) < 2 {
+	if len(records) == 0 {
 		err = fmt.Errorf("no record founded. please add at least one record to csv file")
 		return cmd.ErrorResult(err)
 	}
@@ -112,7 +134,7 @@ func (v *Voucher) createBulkHandler(
 		}
 
 		if notify == "TRUE" {
-			if v.createNotification(vch.Email, vch.Code) != nil {
+			if v.createNotification(vch.Email, vch.Code, vch.Recipient, vch.Amount.ToPAC()) != nil {
 				return cmd.ErrorResult(err)
 			}
 		}
@@ -121,32 +143,33 @@ func (v *Voucher) createBulkHandler(
 	return cmd.SuccessfulResult("Vouchers created successfully!")
 }
 
-func (v *Voucher) createBulkVoucher(records [][]string, callerID uint) ([]*entity.Voucher, error) {
+func (v *Voucher) createBulkVoucher(records []BulkRecorder, callerID uint) ([]*entity.Voucher, error) {
 	vouchers := make([]*entity.Voucher, 0)
-	for rowIndex := 1; rowIndex < len(records); rowIndex++ {
+	for index, record := range records {
 		code := utils.RandomString(8, utils.CapitalAlphanumerical)
 		for _, err := v.db.GetVoucherByCode(code); err == nil; {
 			code = utils.RandomString(8, utils.CapitalAlphanumerical)
 		}
 
-		email := records[rowIndex][0] // TODO: validate email address using regex
-		recipient := records[rowIndex][1]
-		amt, err := amount.FromString(strings.TrimSpace(records[rowIndex][2]))
+		email := record.Email // TODO: validate email address using regex
+		recipient := record.Recipient
+		desc := record.Description
+
+		amt, err := amount.NewAmount(record.Amount)
 		if err != nil {
-			return nil, fmt.Errorf("invalid amount at row %d", rowIndex)
+			return nil, fmt.Errorf("invalid amount at row %d", index+1)
 		}
 
 		maxStake, _ := amount.NewAmount(1000)
 		if amt > maxStake {
 			return nil, fmt.Errorf("stake amount is more than 1000")
 		}
 
-		validMonths, err := strconv.Atoi(strings.TrimSpace(records[rowIndex][3]))
-		if err != nil {
-			return nil, fmt.Errorf("invalid validate months at row %d", rowIndex)
+		validMonths := record.ValidatedInMonth
+		if validMonths < 1 {
+			return nil, fmt.Errorf("num of validated month of code must be greater than 0 at row %d", index+1)
 		}
 
-		desc := records[rowIndex][4]
 		vouchers = append(vouchers, &entity.Voucher{
 			Creator:     callerID,
 			Code:        code,
@@ -161,17 +184,21 @@ func (v *Voucher) createBulkVoucher(records [][]string, callerID uint) ([]*entit
 	return vouchers, nil
 }
 
-func (v *Voucher) createNotification(email, code string) error {
-	notificationData := entity.VoucherNotificationData{Code: code}
+func (v *Voucher) createNotification(email, code, recipient string, amt float64) error {
+	notificationData := entity.VoucherNotificationData{
+		Code:      code,
+		Recipient: recipient,
+		Amount:    amt,
+	}
 	b, err := json.Marshal(notificationData)
 	if err != nil {
 		return err
 	}
 	voucherCodeJSON := datatypes.JSON(b)
 	return v.db.AddNotification(&entity.Notification{
 		Type:      notification.NotificationTypeMail,
+		Status:    entity.NotificationStatusPending,
 		Recipient: email,
 		Data:      voucherCodeJSON,
-		Status:    entity.NotificationStatusPending,
 	})
 }

internal/engine/command/voucher/create_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/h2non/gock"
 	"github.com/pagu-project/Pagu/internal/engine/command"
 	"github.com/pagu-project/Pagu/internal/entity"
 	"github.com/pagu-project/Pagu/internal/repository"
@@ -106,14 +107,30 @@ func TestCreateOne(t *testing.T) {
 
 func TestCreateBulk(t *testing.T) {
 	voucher, db, _, _ := setup(t)
-	t.Run("create and save notification", func(t *testing.T) {
-		db.EXPECT().GetPendingMailNotification().Return(
-			&entity.Notification{}, errors.New(""),
-		).AnyTimes()
+	t.Run("normal", func(t *testing.T) {
+		defer gock.Off()
+		gock.New("http://foo.com").
+			Get("/bar").
+			Reply(200).
+			BodyString("Recipient,Email,Amount,Validated,Description\n" +
+				"foo.bar,a@gmail.com,1,2,Some Descriptions\n" +
+				"foo.bar,b@gmail.com,1,2,Some Descriptions")
+
+		cmd := &command.Command{}
 
+		db.EXPECT().AddVoucher(gomock.Any()).Return(nil).AnyTimes()
 		db.EXPECT().AddNotification(gomock.Any()).Return(nil).AnyTimes()
+		db.EXPECT().GetVoucherByCode(gomock.Any()).Return(
+			entity.Voucher{}, errors.New(""),
+		).AnyTimes()
+
+		args := make(map[string]string)
+		args["file"] = "http://foo.com/bar"
+		args["notify"] = "TRUE"
+		caller := &entity.User{ID: 1}
+		result := voucher.createBulkHandler(caller, cmd, args)
 
-		err := voucher.createNotification("foo@bar", "12345678")
-		assert.Equal(t, nil, err)
+		assert.True(t, result.Successful)
+		assert.Contains(t, result.Message, "Vouchers created successfully!")
 	})
 }

internal/entity/notification.go

@@ -25,5 +25,7 @@ type Notification struct {
 }
 
 type VoucherNotificationData struct {
-	Code string `json:"code"`
+	Code      string  `json:"code"`
+	Amount    float64 `json:"amount"`
+	Recipient string  `json:"recipient"`
 }