From 5ad1c01001d00cf01c5cae84faa340911f7d3373 Mon Sep 17 00:00:00 2001 From: Sean Durkin Date: Sat, 26 Jan 2019 01:27:26 -0500 Subject: [PATCH 1/5] Add attachments to invoices. --- cmswww/api/v1/constants.go | 8 ++ cmswww/api/v1/enums.go | 4 + cmswww/api/v1/route_protocol.go | 8 +- cmswww/cmd/cmswwwcli/client/util.go | 58 ++++++++++-- cmswww/cmd/cmswwwcli/commands/editinvoice.go | 17 +++- .../cmd/cmswwwcli/commands/invoicedetails.go | 15 +++ .../cmd/cmswwwcli/commands/submitinvoice.go | 85 +++++++++++------ cmswww/convert.go | 92 +++++++++++-------- cmswww/database/cockroachdb/cockroachdb.go | 28 ++++++ cmswww/database/cockroachdb/encoding.go | 51 +++++++--- cmswww/database/cockroachdb/models.go | 19 +++- cmswww/database/database.go | 7 +- cmswww/invoice.go | 84 +++++++++++------ cmswww/validate.go | 58 +++++++++--- cmswww/www.go | 2 +- go.mod | 1 + 16 files changed, 399 insertions(+), 138 deletions(-) diff --git a/cmswww/api/v1/constants.go b/cmswww/api/v1/constants.go index 5e2d341..7ea94ca 100644 --- a/cmswww/api/v1/constants.go +++ b/cmswww/api/v1/constants.go @@ -42,6 +42,14 @@ const ( // values for each line item in the CSV. PolicyInvoiceFieldDelimiterChar rune = ',' + // PolicyMaxAttachments is the maximum number of attachments that a + // contractor can submit along with an invoice. + PolicyMaxAttachments = 10 + + // PolicyMaxAttachmentSize is the maximum file size (in bytes) + // accepted for an attachment when creating a new invoice. + PolicyMaxAttachmentSize = 512 * 1024 + // ListPageSize is the maximum number of entries returned // for the routes that return lists ListPageSize = 25 diff --git a/cmswww/api/v1/enums.go b/cmswww/api/v1/enums.go index 508da2d..a056591 100644 --- a/cmswww/api/v1/enums.go +++ b/cmswww/api/v1/enums.go @@ -38,6 +38,8 @@ const ( ErrorStatusMalformedInvoiceFile ErrorStatusT = 27 ErrorStatusInvoicePaymentNotFound ErrorStatusT = 28 ErrorStatusDuplicateInvoice ErrorStatusT = 29 + ErrorStatusMaxAttachmentsExceeded ErrorStatusT = 30 + ErrorStatusMaxAttachmentSizeExceeded ErrorStatusT = 31 // Invoice status codes InvoiceStatusInvalid InvoiceStatusT = 0 // Invalid status @@ -97,6 +99,8 @@ var ( ErrorStatusMalformedInvoiceFile: "malformed invoice file", ErrorStatusInvoicePaymentNotFound: "invoice payment not found", ErrorStatusDuplicateInvoice: "duplicate invoice for this month and year", + ErrorStatusMaxAttachmentsExceeded: "max number of attachments exceeded", + ErrorStatusMaxAttachmentSizeExceeded: "max attachment size exceeded", } // InvoiceStatus converts propsal status codes to human readable text diff --git a/cmswww/api/v1/route_protocol.go b/cmswww/api/v1/route_protocol.go index 1a1aa58..3eb12d5 100644 --- a/cmswww/api/v1/route_protocol.go +++ b/cmswww/api/v1/route_protocol.go @@ -42,6 +42,8 @@ var ( // directory structure must be flattened. The server side SHALL verify MIME // and Digest. type File struct { + Name string `json:"name"` // Filename + MIME string `json:"mime"` // MIME type of file Digest string `json:"digest"` // Digest of unencoded payload Payload string `json:"payload"` // File content, base64 encoded } @@ -70,7 +72,7 @@ type InvoiceRecord struct { Username string `json:"username"` // Username of user who submitted invoice PublicKey string `json:"publickey"` // User's public key, used to verify signature. Signature string `json:"signature"` // Signature of file digest - File *File `json:"file"` // Actual invoice file + Files []File `json:"files"` // Actual invoice file and attachments Version string `json:"version"` // Record version CensorshipRecord CensorshipRecord `json:"censorshiprecord"` @@ -207,7 +209,7 @@ type LogoutReply struct{} type SubmitInvoice struct { Month uint16 `json:"month"` Year uint16 `json:"year"` - File File `json:"file"` // Invoice file + Files []File `json:"files"` // Invoice file and any attachments PublicKey string `json:"publickey"` // Key used to verify signature Signature string `json:"signature"` // Signature of file hash } @@ -220,7 +222,7 @@ type SubmitInvoiceReply struct { // EditInvoice attempts to submit an edit to an existing invoice. type EditInvoice struct { Token string `json:"token"` // Invoice token - File File `json:"file"` // Invoice file + Files []File `json:"files"` // Invoice file and any attachments PublicKey string `json:"publickey"` // Key used to verify signature Signature string `json:"signature"` // Signature of file hash diff --git a/cmswww/cmd/cmswwwcli/client/util.go b/cmswww/cmd/cmswwwcli/client/util.go index 59e1838..ef42c94 100644 --- a/cmswww/cmd/cmswwwcli/client/util.go +++ b/cmswww/cmd/cmswwwcli/client/util.go @@ -1,11 +1,13 @@ package client import ( + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "os" + "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/util" @@ -28,16 +30,56 @@ func invoiceSignature(file v1.File, id *identity.FullIdentity) (string, error) { return hex.EncodeToString(sig[:]), nil } -// verifyProposal verifies the integrity of a proposal by verifying the -// proposal's merkle root (if the files are present), the proposal signature, +// merkleRoot converts the passed in list of files into SHA256 digests, then +// calculates and returns the merkle root of the digests. +func merkleRoot(files []v1.File) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no invoice files found") + } + + digests := make([]*[sha256.Size]byte, len(files)) + for i, f := range files { + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return "", fmt.Errorf("could not convert digest") + } + digests[i] = &d + } + + return hex.EncodeToString(merkle.Root(digests)[:]), nil +} + +// SignMerkleRoot calculates the merkle root of the passed in list of files, +// signs the merkle root with the passed in identity and returns the signature. +func SignMerkleRoot(files []v1.File, id *identity.FullIdentity) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no invoice files found") + } + mr, err := merkleRoot(files) + if err != nil { + return "", err + } + sig := id.SignMessage([]byte(mr)) + return hex.EncodeToString(sig[:]), nil +} + +// VerifyInvoice verifies the integrity of an invoice by verifying the +// invoice's merkle root (if the files are present), the signature, // and the censorship record signature. -func verifyProposal(record v1.InvoiceRecord, serverPubKey string) error { - // Verify the file digest. - if record.File.Digest != record.CensorshipRecord.Merkle { - return fmt.Errorf("digests do not match") +func VerifyInvoice(record v1.InvoiceRecord, serverPubKey string) error { + // Verify merkle root if the invoice files are present. + if len(record.Files) > 0 { + mr, err := merkleRoot(record.Files) + if err != nil { + return err + } + if mr != record.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match; expected %v, got %v", + record.CensorshipRecord.Merkle, mr) + } } - // Verify proposal signature. + // Verify invoice signature. pid, err := util.IdentityFromString(record.PublicKey) if err != nil { return err @@ -47,7 +89,7 @@ func verifyProposal(record v1.InvoiceRecord, serverPubKey string) error { return err } if !pid.VerifyMessage([]byte(record.CensorshipRecord.Merkle), sig) { - return fmt.Errorf("could not verify proposal signature") + return fmt.Errorf("could not verify invoice signature") } // Verify censorship record signature. diff --git a/cmswww/cmd/cmswwwcli/commands/editinvoice.go b/cmswww/cmd/cmswwwcli/commands/editinvoice.go index 4584eb7..2690567 100644 --- a/cmswww/cmd/cmswwwcli/commands/editinvoice.go +++ b/cmswww/cmd/cmswwwcli/commands/editinvoice.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "github.com/decred/contractor-mgmt/cmswww/api/v1" + "github.com/decred/contractor-mgmt/cmswww/cmd/cmswwwcli/client" "github.com/decred/contractor-mgmt/cmswww/cmd/cmswwwcli/config" ) @@ -63,10 +64,10 @@ func (cmd *EditInvoiceCmd) Execute(args []string) error { ei := v1.EditInvoice{ Token: token, - File: v1.File{ + Files: []v1.File{{ Digest: digest, Payload: base64.StdEncoding.EncodeToString(payload), - }, + }}, PublicKey: hex.EncodeToString(id.Public.Key[:]), Signature: hex.EncodeToString(signature[:]), } @@ -77,9 +78,15 @@ func (cmd *EditInvoiceCmd) Execute(args []string) error { return err } - if eir.Invoice.CensorshipRecord.Merkle != digest { - return fmt.Errorf("Digest returned from server did not match client's"+ - " digest: %v %v", digest, eir.Invoice.CensorshipRecord.Merkle) + ir := v1.InvoiceRecord{ + Files: ei.Files, + PublicKey: ei.PublicKey, + Signature: ei.Signature, + CensorshipRecord: eir.Invoice.CensorshipRecord, + } + err = client.VerifyInvoice(ir, config.ServerPublicKey) + if err != nil { + return err } // Store the revision record in case the submitter ever needs it. diff --git a/cmswww/cmd/cmswwwcli/commands/invoicedetails.go b/cmswww/cmd/cmswwwcli/commands/invoicedetails.go index f53fe9a..e42884e 100644 --- a/cmswww/cmd/cmswwwcli/commands/invoicedetails.go +++ b/cmswww/cmd/cmswwwcli/commands/invoicedetails.go @@ -38,6 +38,21 @@ func (cmd *InvoiceDetailsCmd) Execute(args []string) error { fmt.Printf(" Submitted by: %v\n", idr.Invoice.Username) fmt.Printf(" at: %v\n", time.Unix(idr.Invoice.Timestamp, 0)) fmt.Printf(" For: %v\n", date.Format("January 2006")) + + var attachments string + for idx, file := range idr.Invoice.Files { + if idx == 0 { + continue + } + + if idx == 1 { + attachments += fmt.Sprintf("%v %v\n", idx, file.Name) + } else { + attachments += fmt.Sprintf(" %v %v\n", idx, + file.Name) + } + } + fmt.Printf(" Attachments: %v\n", attachments) } return nil diff --git a/cmswww/cmd/cmswwwcli/commands/submitinvoice.go b/cmswww/cmd/cmswwwcli/commands/submitinvoice.go index c0a86b5..424b507 100644 --- a/cmswww/cmd/cmswwwcli/commands/submitinvoice.go +++ b/cmswww/cmd/cmswwwcli/commands/submitinvoice.go @@ -2,7 +2,6 @@ package commands import ( "bufio" - "crypto/sha256" "encoding/base64" "encoding/csv" "encoding/hex" @@ -10,19 +9,25 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "time" + "github.com/decred/politeia/politeiad/api/v1/mime" + "github.com/decred/politeia/util" + "github.com/decred/contractor-mgmt/cmswww/api/v1" + "github.com/decred/contractor-mgmt/cmswww/cmd/cmswwwcli/client" "github.com/decred/contractor-mgmt/cmswww/cmd/cmswwwcli/config" ) type SubmitInvoiceCmd struct { Args struct { - Month string `positional-arg-name:"month"` - Year uint16 `positional-arg-name:"year"` + Attachments []string `positional-arg-name:"attachmentFilepaths"` } `positional-args:"true" optional:"true"` InvoiceFilename string `long:"invoice" optional:"true" description:"Filepath to an invoice CSV"` + Month string `long:"month" optional:"true" description:"Month to specify a prebuilt invoice"` + Year uint16 `long:"year" optional:"true" description:"Year to specify a prebuilt invoice"` } // SubmissionRecord is a record of an invoice submission to the server, @@ -100,12 +105,12 @@ func (cmd *SubmitInvoiceCmd) Execute(args []string) error { } var filename string - if cmd.Args.Month != "" && cmd.Args.Year != 0 { - month, err := ParseMonth(cmd.Args.Month) + if cmd.Month != "" && cmd.Year != 0 { + month, err := ParseMonth(cmd.Month) if err != nil { return err } - year = cmd.Args.Year + year = cmd.Year filename, err = config.GetInvoiceFilename(month, year) if err != nil { @@ -123,43 +128,71 @@ func (cmd *SubmitInvoiceCmd) Execute(args []string) error { return err } + // Read attachment files into memory and convert to type File + files := make([]v1.File, 0, len(cmd.Args.Attachments)+1) + + // Add the invoice file. payload, err := ioutil.ReadFile(filename) if err != nil { - return err + fmt.Errorf("error reading invoice file %v: %v", filename, err) + } + + files = append(files, v1.File{ + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + + for _, path := range cmd.Args.Attachments { + attachment, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading invoice attachment file %v: %v", + path, err) + } + + files = append(files, v1.File{ + Name: filepath.Base(path), + MIME: mime.DetectMimeType(attachment), + Digest: hex.EncodeToString(util.Digest(attachment)), + Payload: base64.StdEncoding.EncodeToString(attachment), + }) } - h := sha256.New() - h.Write(payload) - digest := hex.EncodeToString(h.Sum(nil)) - signature := id.SignMessage([]byte(digest)) + // Compute merkle root and sign it. + sig, err := client.SignMerkleRoot(files, id) + if err != nil { + return fmt.Errorf("SignMerkleRoot: %v", err) + } - ni := v1.SubmitInvoice{ - Month: month, - Year: year, - File: v1.File{ - Digest: digest, - Payload: base64.StdEncoding.EncodeToString(payload), - }, + si := v1.SubmitInvoice{ + Month: month, + Year: year, + Files: files, PublicKey: hex.EncodeToString(id.Public.Key[:]), - Signature: hex.EncodeToString(signature[:]), + Signature: sig, } - var nir v1.SubmitInvoiceReply - err = Ctx.Post(v1.RouteSubmitInvoice, ni, &nir) + var sir v1.SubmitInvoiceReply + err = Ctx.Post(v1.RouteSubmitInvoice, si, &sir) if err != nil { return err } - if nir.CensorshipRecord.Merkle != digest { - return fmt.Errorf("Digest returned from server did not match client's"+ - " digest: %v %v", digest, nir.CensorshipRecord.Merkle) + ir := v1.InvoiceRecord{ + Files: si.Files, + PublicKey: si.PublicKey, + Signature: si.Signature, + CensorshipRecord: sir.CensorshipRecord, + } + err = client.VerifyInvoice(ir, config.ServerPublicKey) + if err != nil { + return err } // Store the submission record in case the submitter ever needs it. submissionRecord := SubmissionRecord{ ServerPublicKey: config.ServerPublicKey, - Submission: ni, - CensorshipRecord: nir.CensorshipRecord, + Submission: si, + CensorshipRecord: sir.CensorshipRecord, } data, err := json.MarshalIndent(submissionRecord, "", " ") if err != nil { diff --git a/cmswww/convert.go b/cmswww/convert.go index 8fa401a..6d954ac 100644 --- a/cmswww/convert.go +++ b/cmswww/convert.go @@ -75,46 +75,68 @@ func convertDatabaseIdentityToIdentity(dbIdentity database.Identity) v1.UserIden } } -func convertInvoiceFileFromWWW(f *v1.File) []pd.File { - return []pd.File{{ - Name: "invoice.csv", - MIME: "text/plain; charset=utf-8", +func convertInvoiceFileToRecordFile(f v1.File) pd.File { + return pd.File{ + Name: f.Name, + MIME: f.MIME, Digest: f.Digest, Payload: f.Payload, - }} + } } -func convertInvoiceCensorFromWWW(f v1.CensorshipRecord) pd.CensorshipRecord { - return pd.CensorshipRecord{ - Token: f.Token, - Merkle: f.Merkle, - Signature: f.Signature, +func convertInvoiceFilesToRecordFiles(files []v1.File) []pd.File { + pdFiles := make([]pd.File, 0, len(files)) + for idx, f := range files { + if idx == 0 { + pdFiles = append(pdFiles, pd.File{ + Name: "invoice.csv", + MIME: "text/plain; charset=utf-8", + Digest: f.Digest, + Payload: f.Payload, + }) + continue + } + + pdFiles = append(pdFiles, convertInvoiceFileToRecordFile(f)) } + return pdFiles } -func convertInvoiceFileFromPD(files []pd.File) *v1.File { - if len(files) == 0 { - return nil +func convertRecordFilesToInvoiceFiles(pdFiles []pd.File) []v1.File { + files := make([]v1.File, 0, len(pdFiles)) + for _, f := range pdFiles { + files = append(files, convertRecordFileToInvoiceFile(f)) } + return files +} - return &v1.File{ - Digest: files[0].Digest, - Payload: files[0].Payload, +func convertRecordFileToInvoiceFile(f pd.File) v1.File { + return v1.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, } } -func convertRecordFilesToDatabaseInvoiceFile(files []pd.File) *database.File { - if len(files) == 0 { - return nil +func convertRecordFilesToDatabaseInvoiceFiles(pdFiles []pd.File) []database.InvoiceFile { + files := make([]database.InvoiceFile, 0, len(pdFiles)) + for _, pdFile := range pdFiles { + files = append(files, convertRecordFileToDatabaseInvoiceFile(pdFile)) } + return files +} - return &database.File{ - Digest: files[0].Digest, - Payload: files[0].Payload, +func convertRecordFileToDatabaseInvoiceFile(f pd.File) database.InvoiceFile { + return database.InvoiceFile{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, } } -func convertInvoiceCensorFromPD(f pd.CensorshipRecord) v1.CensorshipRecord { +func convertRecordCensorToInvoiceCensor(f pd.CensorshipRecord) v1.CensorshipRecord { return v1.CensorshipRecord{ Token: f.Token, Merkle: f.Merkle, @@ -124,7 +146,7 @@ func convertInvoiceCensorFromPD(f pd.CensorshipRecord) v1.CensorshipRecord { func (c *cmswww) convertRecordToDatabaseInvoice(p pd.Record) (*database.Invoice, error) { dbInvoice := database.Invoice{ - File: convertRecordFilesToDatabaseInvoiceFile(p.Files), + Files: convertRecordFilesToDatabaseInvoiceFiles(p.Files), Token: p.CensorshipRecord.Token, ServerSignature: p.CensorshipRecord.Signature, Version: p.Version, @@ -254,21 +276,19 @@ func convertDatabaseInvoiceToInvoice(dbInvoice *database.Invoice) *v1.InvoiceRec invoice.PublicKey = dbInvoice.PublicKey invoice.Signature = dbInvoice.UserSignature invoice.Version = dbInvoice.Version - if dbInvoice.File != nil { - invoice.File = &v1.File{ - Digest: dbInvoice.File.Digest, - Payload: dbInvoice.File.Payload, - } - } invoice.CensorshipRecord = v1.CensorshipRecord{ - Token: dbInvoice.Token, - //Merkle: dbInvoice.File.Digest, + Token: dbInvoice.Token, + Merkle: dbInvoice.MerkleRoot, Signature: dbInvoice.ServerSignature, } - // TODO: clean up, merkle should always be set - if dbInvoice.File != nil { - invoice.CensorshipRecord.Merkle = dbInvoice.File.Digest + for _, f := range dbInvoice.Files { + invoice.Files = append(invoice.Files, v1.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + }) } return &invoice @@ -282,7 +302,7 @@ func convertDatabaseInvoicesToInvoices(dbInvoices []database.Invoice) []v1.Invoi return invoices } -func convertErrorStatusFromPD(s int) v1.ErrorStatusT { +func convertPDErrorStatusToErrorStatus(s int) v1.ErrorStatusT { switch pd.ErrorStatusT(s) { case pd.ErrorStatusInvalidFileDigest: return v1.ErrorStatusInvalidFileDigest diff --git a/cmswww/database/cockroachdb/cockroachdb.go b/cmswww/database/cockroachdb/cockroachdb.go index 4bf1020..8d7db3a 100644 --- a/cmswww/database/cockroachdb/cockroachdb.go +++ b/cmswww/database/cockroachdb/cockroachdb.go @@ -257,6 +257,12 @@ func (c *cockroachdb) GetInvoiceByToken(token string) (*database.Invoice, error) return nil, result.Error } + result = c.db.Where("invoice_token = ?", invoice.Token).Find( + &invoice.Files) + if result.Error != nil { + return nil, result.Error + } + result = c.db.Where("invoice_token = ?", invoice.Token).Find( &invoice.Payments) if result.Error != nil { @@ -345,6 +351,22 @@ func (c *cockroachdb) UpdateInvoicePayment(dbInvoicePayment *database.InvoicePay return c.db.Save(invoicePayment).Error } +func (c *cockroachdb) CreateInvoiceFiles(token string, dbInvoiceFiles []database.InvoiceFile) error { + log.Debugf("CreateInvoiceFiles: %v", token) + + for _, dbInvoiceFile := range dbInvoiceFiles { + invoiceFile := EncodeInvoiceFile(&dbInvoiceFile) + invoiceFile.InvoiceToken = token + + err := c.db.Create(invoiceFile).Error + if err != nil { + return err + } + } + + return nil +} + // Deletes all data from all tables. // // DeleteAllData satisfies the backend interface. @@ -417,6 +439,11 @@ func New(dataDir, dbName, username, host string) (*cockroachdb, error) { return nil, fmt.Errorf("error dropping %v table: %v", tableNameInvoicePayment, err) } + err = c.dropTable(tableNameInvoiceFile) + if err != nil { + return nil, fmt.Errorf("error dropping %v table: %v", + tableNameInvoiceFile, err) + } err = c.dropTable(tableNameInvoice) if err != nil { return nil, fmt.Errorf("error dropping %v table: %v", tableNameInvoice, @@ -427,6 +454,7 @@ func New(dataDir, dbName, username, host string) (*cockroachdb, error) { &User{}, &Identity{}, &Invoice{}, + &InvoiceFile{}, &InvoiceChange{}, &InvoicePayment{}, ) diff --git a/cmswww/database/cockroachdb/encoding.go b/cmswww/database/cockroachdb/encoding.go index 3bad722..d0fc98d 100644 --- a/cmswww/database/cockroachdb/encoding.go +++ b/cmswww/database/cockroachdb/encoding.go @@ -211,17 +211,19 @@ func EncodeInvoice(dbInvoice *database.Invoice) *Invoice { invoice.Status = uint(dbInvoice.Status) invoice.StatusChangeReason = dbInvoice.StatusChangeReason invoice.Timestamp = time.Unix(dbInvoice.Timestamp, 0) - if dbInvoice.File != nil { - invoice.FilePayload = dbInvoice.File.Payload - invoice.FileMIME = dbInvoice.File.MIME - invoice.FileDigest = dbInvoice.File.Digest - } invoice.PublicKey = dbInvoice.PublicKey invoice.UserSignature = dbInvoice.UserSignature invoice.ServerSignature = dbInvoice.ServerSignature + invoice.MerkleRoot = dbInvoice.MerkleRoot invoice.Proposal = dbInvoice.Proposal invoice.Version = dbInvoice.Version + for _, dbInvoiceFile := range dbInvoice.Files { + invoiceFile := EncodeInvoiceFile(&dbInvoiceFile) + invoiceFile.InvoiceToken = invoice.Token + invoice.Files = append(invoice.Files, *invoiceFile) + } + for _, dbInvoiceChange := range dbInvoice.Changes { invoiceChange := EncodeInvoiceChange(&dbInvoiceChange) invoiceChange.InvoiceToken = invoice.Token @@ -238,6 +240,19 @@ func EncodeInvoice(dbInvoice *database.Invoice) *Invoice { return &invoice } +// EncodeInvoiceFile encodes a generic database.InvoiceFile instance into a cockroachdb +// InvoiceFile. +func EncodeInvoiceFile(dbInvoiceFile *database.InvoiceFile) *InvoiceFile { + invoiceFile := InvoiceFile{} + + invoiceFile.Name = dbInvoiceFile.Name + invoiceFile.MIME = dbInvoiceFile.MIME + invoiceFile.Digest = dbInvoiceFile.Digest + invoiceFile.Payload = dbInvoiceFile.Payload + + return &invoiceFile +} + // EncodeInvoiceChange encodes a generic database.InvoiceChange instance into a cockroachdb // InvoiceChange. func EncodeInvoiceChange(dbInvoiceChange *database.InvoiceChange) *InvoiceChange { @@ -279,18 +294,17 @@ func DecodeInvoice(invoice *Invoice) (*database.Invoice, error) { dbInvoice.Status = v1.InvoiceStatusT(invoice.Status) dbInvoice.StatusChangeReason = invoice.StatusChangeReason dbInvoice.Timestamp = invoice.Timestamp.Unix() - if invoice.FilePayload != "" { - dbInvoice.File = &database.File{ - Payload: invoice.FilePayload, - MIME: invoice.FileMIME, - Digest: invoice.FileDigest, - } - } dbInvoice.PublicKey = invoice.PublicKey dbInvoice.UserSignature = invoice.UserSignature dbInvoice.ServerSignature = invoice.ServerSignature + dbInvoice.MerkleRoot = invoice.MerkleRoot dbInvoice.Proposal = invoice.Proposal dbInvoice.Version = invoice.Version + + for _, invoiceFile := range invoice.Files { + dbInvoiceFile := DecodeInvoiceFile(&invoiceFile) + dbInvoice.Files = append(dbInvoice.Files, *dbInvoiceFile) + } /* for _, invoiceChange := range invoice.Changes { dbInvoiceChange := DecodeInvoiceChange(&invoiceChange) @@ -305,6 +319,19 @@ func DecodeInvoice(invoice *Invoice) (*database.Invoice, error) { return &dbInvoice, nil } +// DecodeInvoiceFile decodes a cockroachdb InvoiceFile instance into a generic +// database.InvoiceFile. +func DecodeInvoiceFile(invoiceFile *InvoiceFile) *database.InvoiceFile { + dbInvoiceFile := database.InvoiceFile{} + + dbInvoiceFile.Name = invoiceFile.Name + dbInvoiceFile.MIME = invoiceFile.MIME + dbInvoiceFile.Digest = invoiceFile.Digest + dbInvoiceFile.Payload = invoiceFile.Payload + + return &dbInvoiceFile +} + // DecodeInvoiceChange decodes a cockroachdb InvoiceChange instance into a generic // database.InvoiceChange. func DecodeInvoiceChange(invoiceChange *InvoiceChange) *database.InvoiceChange { diff --git a/cmswww/database/cockroachdb/models.go b/cmswww/database/cockroachdb/models.go index 8af20d5..8f2f85d 100644 --- a/cmswww/database/cockroachdb/models.go +++ b/cmswww/database/cockroachdb/models.go @@ -13,6 +13,7 @@ const ( tableNameUser = "users" tableNameIdentity = "identities" tableNameInvoice = "invoices" + tableNameInvoiceFile = "invoice_files" tableNameInvoiceChange = "invoice_changes" tableNameInvoicePayment = "invoice_payments" ) @@ -68,15 +69,14 @@ type Invoice struct { Timestamp time.Time `gorm:"not_null"` Status uint `gorm:"not_null"` StatusChangeReason string - FilePayload string `gorm:"type:text"` - FileMIME string - FileDigest string PublicKey string `gorm:"not_null"` UserSignature string `gorm:"not_null"` ServerSignature string `gorm:"not_null"` + MerkleRoot string `gorm:"not_null"` Proposal string Version string + Files []InvoiceFile Changes []InvoiceChange Payments []InvoicePayment @@ -90,6 +90,19 @@ func (i Invoice) TableName() string { return tableNameInvoice } +type InvoiceFile struct { + gorm.Model + InvoiceToken string + Name string + MIME string + Digest string + Payload string `gorm:"type:text"` +} + +func (i InvoiceFile) TableName() string { + return tableNameInvoiceFile +} + type InvoiceChange struct { gorm.Model InvoiceToken string diff --git a/cmswww/database/database.go b/cmswww/database/database.go index ec2503e..ae3745a 100644 --- a/cmswww/database/database.go +++ b/cmswww/database/database.go @@ -56,6 +56,7 @@ type Database interface { GetInvoiceByToken(string) (*Invoice, error) // Return invoice given its token GetInvoices(InvoicesRequest) ([]Invoice, int, error) // Return a list of invoices UpdateInvoicePayment(*InvoicePayment) error // Update an existing invoice's payment + CreateInvoiceFiles(string, []InvoiceFile) error // Create invoice files DeleteAllData() error // Delete all data from all tables @@ -108,18 +109,20 @@ type Invoice struct { Timestamp int64 Status v1.InvoiceStatusT StatusChangeReason string - File *File PublicKey string UserSignature string ServerSignature string + MerkleRoot string Proposal string // Optional link to a Politeia proposal Version string // Version number of this invoice + Files []InvoiceFile Changes []InvoiceChange Payments []InvoicePayment } -type File struct { +type InvoiceFile struct { + Name string Payload string MIME string Digest string diff --git a/cmswww/invoice.go b/cmswww/invoice.go index ea996f9..35a1394 100644 --- a/cmswww/invoice.go +++ b/cmswww/invoice.go @@ -77,6 +77,26 @@ func validateStatusTransition( return nil } +func (c *cmswww) validateInvoiceUnique(user *database.User, month, year uint16) error { + invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ + UserID: strconv.FormatUint(user.ID, 10), + Month: month, + Year: year, + }) + if err != nil { + return err + } + + if len(invoices) > 0 { + return v1.UserError{ + ErrorCode: v1.ErrorStatusDuplicateInvoice, + ErrorContext: []string{invoices[0].Token}, + } + } + + return nil +} + func (c *cmswww) refreshExistingInvoicePayments(dbInvoice *database.Invoice) error { for _, dbInvoicePayment := range dbInvoice.Payments { if dbInvoicePayment.TxID != "" { @@ -101,7 +121,7 @@ func (c *cmswww) deriveTotalCostFromInvoice( dbInvoice *database.Invoice, invoicePayment *v1.InvoicePayment, ) error { - b, err := base64.StdEncoding.DecodeString(dbInvoice.File.Payload) + b, err := base64.StdEncoding.DecodeString(dbInvoice.Files[0].Payload) if err != nil { return err } @@ -148,7 +168,7 @@ func (c *cmswww) createInvoiceReview(invoice *database.Invoice) (*v1.InvoiceRevi LineItems: make([]v1.InvoiceReviewLineItem, 0), } - b, err := base64.StdEncoding.DecodeString(invoice.File.Payload) + b, err := base64.StdEncoding.DecodeString(invoice.Files[0].Payload) if err != nil { return nil, err } @@ -395,7 +415,7 @@ func (c *cmswww) updateInvoicePayment( } func (c *cmswww) fetchInvoiceFileIfNecessary(invoice *database.Invoice) error { - if invoice.File != nil { + if len(invoice.Files) == 0 { return nil } @@ -426,7 +446,7 @@ func (c *cmswww) fetchInvoiceFileIfNecessary(invoice *database.Invoice) error { return err } - invoice.File = convertRecordFilesToDatabaseInvoiceFile(pdReply.Record.Files) + invoice.Files = convertRecordFilesToDatabaseInvoiceFiles(pdReply.Record.Files) return nil } @@ -800,6 +820,14 @@ func (c *cmswww) HandleInvoiceDetails( return nil, err } + // If the files are already loaded for this invoice, return the version + // from the database; no need to make a request to Politeiad. + if len(invoice.Files) > 0 { + idr.Invoice = *invoice + return &idr, nil + } + + // Fetch the full record from Politeiad. responseBody, err := c.rpc(http.MethodPost, pd.GetVettedRoute, pd.GetVetted{ Token: id.Token, @@ -823,8 +851,15 @@ func (c *cmswww) HandleInvoiceDetails( return nil, err } - invoice.File = convertInvoiceFileFromPD(pdReply.Record.Files) - invoice.Username = c.getUsernameByID(invoice.UserID) + invoice.Files = convertRecordFilesToInvoiceFiles(pdReply.Record.Files) + + // Update the database with the files fetched from Politeiad. + err = c.db.CreateInvoiceFiles(dbInvoice.Token, + convertRecordFilesToDatabaseInvoiceFiles(pdReply.Record.Files)) + if err != nil { + return nil, err + } + idr.Invoice = *invoice return &idr, nil } @@ -836,30 +871,19 @@ func (c *cmswww) HandleSubmitInvoice( w http.ResponseWriter, r *http.Request, ) (interface{}, error) { - ni := req.(*v1.SubmitInvoice) - fmt.Println(ni) - err := validateInvoice(ni.Signature, ni.PublicKey, ni.File.Payload, - int(ni.Month), int(ni.Year), user) + si := req.(*v1.SubmitInvoice) + + err := validateInvoice(si.Signature, si.PublicKey, si.Files, + int(si.Month), int(si.Year), user) if err != nil { return nil, err } - invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ - UserID: strconv.FormatUint(user.ID, 10), - Month: ni.Month, - Year: ni.Year, - }) + err = c.validateInvoiceUnique(user, si.Month, si.Year) if err != nil { return nil, err } - if len(invoices) > 0 { - return nil, v1.UserError{ - ErrorCode: v1.ErrorStatusDuplicateInvoice, - ErrorContext: []string{invoices[0].Token}, - } - } - var nir v1.SubmitInvoiceReply challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -869,12 +893,12 @@ func (c *cmswww) HandleSubmitInvoice( // Assemble metadata record ts := time.Now().Unix() md, err := json.Marshal(BackendInvoiceMetadata{ - Month: ni.Month, - Year: ni.Year, Version: VersionBackendInvoiceMetadata, + Month: si.Month, + Year: si.Year, Timestamp: ts, - PublicKey: ni.PublicKey, - Signature: ni.Signature, + PublicKey: si.PublicKey, + Signature: si.Signature, }) if err != nil { return nil, err @@ -886,7 +910,7 @@ func (c *cmswww) HandleSubmitInvoice( ID: mdStreamGeneral, Payload: string(md), }}, - Files: convertInvoiceFileFromWWW(&ni.File), + Files: convertInvoiceFilesToRecordFiles(si.Files), } var pdNewRecordReply pd.NewRecordReply @@ -968,7 +992,7 @@ func (c *cmswww) HandleSubmitInvoice( return nil, err } - nir.CensorshipRecord = convertInvoiceCensorFromPD( + nir.CensorshipRecord = convertRecordCensorToInvoiceCensor( pdNewRecordReply.CensorshipRecord) return &nir, nil } @@ -987,7 +1011,7 @@ func (c *cmswww) HandleEditInvoice( return nil, err } - err = validateInvoice(ei.Signature, ei.PublicKey, ei.File.Payload, + err = validateInvoice(ei.Signature, ei.PublicKey, ei.Files, int(dbInvoice.Month), int(dbInvoice.Year), user) if err != nil { return nil, err @@ -1032,7 +1056,7 @@ func (c *cmswww) HandleEditInvoice( ID: mdStreamChanges, Payload: string(changes), }}, - FilesAdd: convertInvoiceFileFromWWW(&ei.File), + FilesAdd: convertInvoiceFilesToRecordFiles(ei.Files), } var pdUpdateRecordReply pd.UpdateRecordReply diff --git a/cmswww/validate.go b/cmswww/validate.go index a587bae..103cb0f 100644 --- a/cmswww/validate.go +++ b/cmswww/validate.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "crypto/sha256" "encoding/base64" "encoding/csv" "encoding/hex" @@ -11,6 +12,7 @@ import ( "strings" "time" + "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/util" @@ -94,7 +96,8 @@ func checkSignature(id []byte, signature string, elements ...string) error { } func validateInvoice( - signature, publicKey, payload string, + signature, publicKey string, + files []v1.File, month, year int, user *database.User, ) error { @@ -119,21 +122,52 @@ func validateInvoice( return err } - // Check for the presence of the file. - if payload == "" { + if len(files) == 0 { return v1.UserError{ ErrorCode: v1.ErrorStatusInvalidInput, } } - data, err := base64.StdEncoding.DecodeString(payload) - if err != nil { - return err + if len(files)-1 > v1.PolicyMaxAttachments { + return v1.UserError{ + ErrorCode: v1.ErrorStatusMaxAttachmentsExceeded, + } + } + + var invoiceData []byte + var hashes []*[sha256.Size]byte + for idx, file := range files { + // Check for the presence of the file. + if file.Payload == "" || file.Digest == "" { + return v1.UserError{ + ErrorCode: v1.ErrorStatusInvalidInput, + } + } + + data, err := base64.StdEncoding.DecodeString(file.Payload) + if err != nil { + return err + } + if idx == 0 { + invoiceData = data + } + + if len(data) > v1.PolicyMaxAttachmentSize { + return v1.UserError{ + ErrorCode: v1.ErrorStatusMaxAttachmentSizeExceeded, + } + } + + // Append digest to array for merkle root calculation + digest := util.Digest(data) + var d [sha256.Size]byte + copy(d[:], digest) + hashes = append(hashes, &d) } - digest := util.Digest(data) - // Validate the string representation of the digest against the signature. - if !pk.VerifyMessage([]byte(hex.EncodeToString(digest)), sig) { + // Note that we need validate the string representation of the merkle + mr := merkle.Root(hashes) + if !pk.VerifyMessage([]byte(hex.EncodeToString(mr[:])), sig) { return v1.UserError{ ErrorCode: v1.ErrorStatusInvalidSignature, } @@ -143,15 +177,15 @@ func validateInvoice( t := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) str := fmt.Sprintf("%v %v", v1.PolicyInvoiceCommentChar, t.Format("2006-01")) - if strings.HasPrefix(string(data), str) || - strings.Contains(string(data), "\n"+str) { + if strings.HasPrefix(string(invoiceData), str) || + strings.Contains(string(invoiceData), "\n"+str) { return v1.UserError{ ErrorCode: v1.ErrorStatusMalformedInvoiceFile, } } // Validate that the invoice is CSV-formatted. - csvReader := csv.NewReader(strings.NewReader(string(data))) + csvReader := csv.NewReader(strings.NewReader(string(invoiceData))) csvReader.Comma = v1.PolicyInvoiceFieldDelimiterChar csvReader.Comment = v1.PolicyInvoiceCommentChar csvReader.TrimLeadingSpace = true diff --git a/cmswww/www.go b/cmswww/www.go index c7020d3..3e054a3 100644 --- a/cmswww/www.go +++ b/cmswww/www.go @@ -110,7 +110,7 @@ func RespondWithError( } if pdError, ok := args[0].(v1.PDError); ok { - pdErrorCode := convertErrorStatusFromPD(pdError.ErrorReply.ErrorCode) + pdErrorCode := convertPDErrorStatusToErrorStatus(pdError.ErrorReply.ErrorCode) pdErrorStatus := pd.ErrorStatus[pd.ErrorStatusT(pdError.ErrorReply.ErrorCode)] if pdErrorCode == v1.ErrorStatusInvalid { errorCode := time.Now().Unix() diff --git a/go.mod b/go.mod index 2bf7aed..3b25576 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/decred/dcrd/chaincfg v1.2.1 github.com/decred/dcrd/dcrutil v1.2.0 github.com/decred/dcrd/wire v1.2.0 + github.com/decred/dcrtime v0.0.0-20180808181920-4c91c4cbed09 github.com/decred/dcrwallet v1.2.2 github.com/decred/politeia v0.0.0-20190114033329-793ab1b7b1e9 github.com/decred/slog v1.0.0 From a3550d31c8082879f00377424ef8499358f09f37 Mon Sep 17 00:00:00 2001 From: Sean Durkin Date: Mon, 28 Jan 2019 23:07:02 -0500 Subject: [PATCH 2/5] Fix issue that travis identified. --- cmswww/cmd/cmswwwcli/commands/submitinvoice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmswww/cmd/cmswwwcli/commands/submitinvoice.go b/cmswww/cmd/cmswwwcli/commands/submitinvoice.go index 424b507..a947c6c 100644 --- a/cmswww/cmd/cmswwwcli/commands/submitinvoice.go +++ b/cmswww/cmd/cmswwwcli/commands/submitinvoice.go @@ -134,7 +134,7 @@ func (cmd *SubmitInvoiceCmd) Execute(args []string) error { // Add the invoice file. payload, err := ioutil.ReadFile(filename) if err != nil { - fmt.Errorf("error reading invoice file %v: %v", filename, err) + return fmt.Errorf("error reading invoice file %v: %v", filename, err) } files = append(files, v1.File{ From 92d6c43285a6028e8affba138f664966ed2b5372 Mon Sep 17 00:00:00 2001 From: Sean Durkin Date: Thu, 31 Jan 2019 00:12:39 -0500 Subject: [PATCH 3/5] Fix versioning for invoices. --- cmswww/cmd/cmswwwcli/client/util.go | 2 +- cmswww/cmd/cmswwwdataload/main.go | 2 + cmswww/convert.go | 2 +- cmswww/database/cockroachdb/cockroachdb.go | 36 ++++- cmswww/database/cockroachdb/encoding.go | 4 +- cmswww/database/cockroachdb/models.go | 29 ++-- cmswww/database/database.go | 32 ++--- cmswww/inventory.go | 12 -- cmswww/invoice.go | 160 ++++++++++++--------- 9 files changed, 159 insertions(+), 120 deletions(-) diff --git a/cmswww/cmd/cmswwwcli/client/util.go b/cmswww/cmd/cmswwwcli/client/util.go index ef42c94..4b05f15 100644 --- a/cmswww/cmd/cmswwwcli/client/util.go +++ b/cmswww/cmd/cmswwwcli/client/util.go @@ -75,7 +75,7 @@ func VerifyInvoice(record v1.InvoiceRecord, serverPubKey string) error { } if mr != record.CensorshipRecord.Merkle { return fmt.Errorf("merkle roots do not match; expected %v, got %v", - record.CensorshipRecord.Merkle, mr) + mr, record.CensorshipRecord.Merkle) } } diff --git a/cmswww/cmd/cmswwwdataload/main.go b/cmswww/cmd/cmswwwdataload/main.go index 40a6a37..a900288 100644 --- a/cmswww/cmd/cmswwwdataload/main.go +++ b/cmswww/cmd/cmswwwdataload/main.go @@ -444,6 +444,7 @@ func deleteExistingData() error { func stopPoliteiad() { if politeiadCmd != nil { fmt.Printf("Stopping politeiad\n") + pdLogFile.Sync() politeiadCmd.Process.Kill() politeiadCmd = nil } @@ -452,6 +453,7 @@ func stopPoliteiad() { func stopCmswww() { if cmswwwCmd != nil { fmt.Printf("Stopping cmswww\n") + cmswwwLogFile.Sync() cmswwwCmd.Process.Kill() cmswwwCmd = nil } diff --git a/cmswww/convert.go b/cmswww/convert.go index 6d954ac..65a4370 100644 --- a/cmswww/convert.go +++ b/cmswww/convert.go @@ -149,6 +149,7 @@ func (c *cmswww) convertRecordToDatabaseInvoice(p pd.Record) (*database.Invoice, Files: convertRecordFilesToDatabaseInvoiceFiles(p.Files), Token: p.CensorshipRecord.Token, ServerSignature: p.CensorshipRecord.Signature, + MerkleRoot: p.CensorshipRecord.Merkle, Version: p.Version, } for _, m := range p.Metadata { @@ -245,7 +246,6 @@ func convertStreamPaymentToDatabaseInvoicePayment(mdPayment BackendInvoiceMDPaym func convertDatabaseInvoicePaymentsToStreamPayments(dbInvoice *database.Invoice) (string, error) { mdPayments := "" for _, dbInvoicePayment := range dbInvoice.Payments { - log.Infof("address from loop: %v", dbInvoicePayment.Address) mdPayment, err := json.Marshal(BackendInvoiceMDPayment{ Version: VersionBackendInvoiceMDPayment, IsTotalCost: dbInvoicePayment.IsTotalCost, diff --git a/cmswww/database/cockroachdb/cockroachdb.go b/cmswww/database/cockroachdb/cockroachdb.go index 8d7db3a..8523fda 100644 --- a/cmswww/database/cockroachdb/cockroachdb.go +++ b/cmswww/database/cockroachdb/cockroachdb.go @@ -246,9 +246,20 @@ func (c *cockroachdb) UpdateInvoice(dbInvoice *database.Invoice) error { func (c *cockroachdb) GetInvoiceByToken(token string) (*database.Invoice, error) { log.Debugf("GetInvoiceByToken: %v", token) + tbl := fmt.Sprintf("%v i", tableNameInvoice) + sel := "i.*, u.username" + joins := fmt.Sprintf( + "inner join %v u on i.user_id = u.id "+ + "inner join ("+ + "select token, max(version) version from %v group by token"+ + ") i2 "+ + "on ("+ + "i2.token = i.token and i2.version = i.version"+ + ")", + tableNameUser, tableNameInvoice) + var invoice Invoice - result := c.db.Table(fmt.Sprintf("%v i", tableNameInvoice)).Select("i.*, u.username").Joins( - "inner join users u on i.user_id = u.id").Where( + result := c.db.Table(tbl).Select(sel).Joins(joins).Where( "i.token = ?", token).Scan(&invoice) if result.Error != nil { if gorm.IsRecordNotFoundError(result.Error) { @@ -303,7 +314,15 @@ func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]d tbl := fmt.Sprintf("%v i", tableNameInvoice) sel := "i.*, u.username" - join := fmt.Sprintf("inner join %v u on i.user_id = u.id", tableNameUser) + joins := fmt.Sprintf( + "inner join %v u on i.user_id = u.id "+ + "inner join ("+ + "select token, max(version) version from %v group by token"+ + ") i2 "+ + "on ("+ + "i2.token = i.token and i2.version = i.version"+ + ")", + tableNameUser, tableNameInvoice) order := "i.timestamp asc" db := c.db.Table(tbl) @@ -311,7 +330,7 @@ func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]d offset := invoicesRequest.Page * v1.ListPageSize db = db.Offset(offset).Limit(v1.ListPageSize) } - db = db.Select(sel).Joins(join) + db = db.Select(sel).Joins(joins) db = c.addWhereClause(db, paramsMap) db = db.Order(order) @@ -324,11 +343,11 @@ func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]d return nil, 0, result.Error } - // If the number of users returned equals the apage size, + // If the number of users returned equals the page size, // find the count of all users that match the query. numMatches := len(invoices) if len(invoices) == v1.ListPageSize { - db = c.db.Table(tbl).Select(sel).Joins(join) + db = c.db.Table(tbl).Select(sel).Joins(joins) db = c.addWhereClause(db, paramsMap) result = db.Count(&numMatches) if result.Error != nil { @@ -351,7 +370,10 @@ func (c *cockroachdb) UpdateInvoicePayment(dbInvoicePayment *database.InvoicePay return c.db.Save(invoicePayment).Error } -func (c *cockroachdb) CreateInvoiceFiles(token string, dbInvoiceFiles []database.InvoiceFile) error { +func (c *cockroachdb) CreateInvoiceFiles( + token, version string, + dbInvoiceFiles []database.InvoiceFile, +) error { log.Debugf("CreateInvoiceFiles: %v", token) for _, dbInvoiceFile := range dbInvoiceFiles { diff --git a/cmswww/database/cockroachdb/encoding.go b/cmswww/database/cockroachdb/encoding.go index d0fc98d..98ae35e 100644 --- a/cmswww/database/cockroachdb/encoding.go +++ b/cmswww/database/cockroachdb/encoding.go @@ -271,7 +271,7 @@ func EncodeInvoicePayment(dbInvoicePayment *database.InvoicePayment) *InvoicePay invoicePayment := InvoicePayment{} invoicePayment.ID = uint(dbInvoicePayment.ID) - invoicePayment.InvoiceToken = dbInvoicePayment.InvoiceToken + //invoicePayment.InvoiceToken = dbInvoicePayment.InvoiceToken invoicePayment.IsTotalCost = dbInvoicePayment.IsTotalCost invoicePayment.Address = dbInvoicePayment.Address invoicePayment.Amount = uint(dbInvoicePayment.Amount) @@ -350,7 +350,7 @@ func DecodeInvoicePayment(invoicePayment *InvoicePayment) *database.InvoicePayme dbInvoicePayment := database.InvoicePayment{} dbInvoicePayment.ID = uint64(invoicePayment.ID) - dbInvoicePayment.InvoiceToken = invoicePayment.InvoiceToken + //dbInvoicePayment.InvoiceToken = invoicePayment.InvoiceToken dbInvoicePayment.IsTotalCost = invoicePayment.IsTotalCost dbInvoicePayment.Address = invoicePayment.Address dbInvoicePayment.Amount = uint64(invoicePayment.Amount) diff --git a/cmswww/database/cockroachdb/models.go b/cmswww/database/cockroachdb/models.go index 8f2f85d..77fbb87 100644 --- a/cmswww/database/cockroachdb/models.go +++ b/cmswww/database/cockroachdb/models.go @@ -62,6 +62,7 @@ func (i Identity) TableName() string { type Invoice struct { Token string `gorm:"primary_key"` + Version string `gorm:"primary_key"` UserID uint `gorm:"not_null"` Username string `gorm:"-"` // Only populated when reading from the database Month uint `gorm:"not_null"` @@ -74,7 +75,6 @@ type Invoice struct { ServerSignature string `gorm:"not_null"` MerkleRoot string `gorm:"not_null"` Proposal string - Version string Files []InvoiceFile Changes []InvoiceChange @@ -92,11 +92,12 @@ func (i Invoice) TableName() string { type InvoiceFile struct { gorm.Model - InvoiceToken string - Name string - MIME string - Digest string - Payload string `gorm:"type:text"` + InvoiceToken string + InvoiceVersion string + Name string + MIME string + Digest string + Payload string `gorm:"type:text"` } func (i InvoiceFile) TableName() string { @@ -106,6 +107,7 @@ func (i InvoiceFile) TableName() string { type InvoiceChange struct { gorm.Model InvoiceToken string + InvoiceVersion string AdminPublicKey string NewStatus uint Timestamp time.Time @@ -117,13 +119,14 @@ func (i InvoiceChange) TableName() string { type InvoicePayment struct { gorm.Model - InvoiceToken string - IsTotalCost bool `gorm:"not_null"` - Address string `gorm:"not_null"` - Amount uint `gorm:"not_null"` - TxNotBefore int64 `gorm:"not_null"` - PollExpiry int64 - TxID string + InvoiceToken string + InvoiceVersion string + IsTotalCost bool `gorm:"not_null"` + Address string `gorm:"not_null"` + Amount uint `gorm:"not_null"` + TxNotBefore int64 `gorm:"not_null"` + PollExpiry int64 + TxID string } func (i InvoicePayment) TableName() string { diff --git a/cmswww/database/database.go b/cmswww/database/database.go index ae3745a..dcd8b51 100644 --- a/cmswww/database/database.go +++ b/cmswww/database/database.go @@ -51,12 +51,12 @@ type Database interface { GetUsers(username string, page int) ([]User, int, error) // Returns a list of users and total count that match the provided username. // Invoice functions - CreateInvoice(*Invoice) error // Create new invoice - UpdateInvoice(*Invoice) error // Update existing invoice - GetInvoiceByToken(string) (*Invoice, error) // Return invoice given its token - GetInvoices(InvoicesRequest) ([]Invoice, int, error) // Return a list of invoices - UpdateInvoicePayment(*InvoicePayment) error // Update an existing invoice's payment - CreateInvoiceFiles(string, []InvoiceFile) error // Create invoice files + CreateInvoice(*Invoice) error // Create new invoice + UpdateInvoice(*Invoice) error // Update existing invoice + GetInvoiceByToken(string) (*Invoice, error) // Return invoice given its token + GetInvoices(InvoicesRequest) ([]Invoice, int, error) // Return a list of invoices + UpdateInvoicePayment(*InvoicePayment) error // Update an existing invoice's payment + CreateInvoiceFiles(string, string, []InvoiceFile) error // Create invoice files DeleteAllData() error // Delete all data from all tables @@ -100,8 +100,10 @@ type Identity struct { Deactivated int64 } +// Invoice represents an invoice submitted by a contractor for payment. type Invoice struct { - Token string + Token string // Unique id for this invoice + Version string // Version number of this invoice UserID uint64 Username string // Only populated when reading from the database Month uint16 @@ -114,7 +116,6 @@ type Invoice struct { ServerSignature string MerkleRoot string Proposal string // Optional link to a Politeia proposal - Version string // Version number of this invoice Files []InvoiceFile Changes []InvoiceChange @@ -136,14 +137,13 @@ type InvoiceChange struct { } type InvoicePayment struct { - ID uint64 - InvoiceToken string - IsTotalCost bool // Whether this payment represents the total cost of the invoice - Address string - Amount uint64 - TxNotBefore int64 - PollExpiry int64 - TxID string + ID uint64 + IsTotalCost bool // Whether this payment represents the total cost of the invoice + Address string + Amount uint64 + TxNotBefore int64 + PollExpiry int64 + TxID string } func (id *Identity) IsActive() bool { diff --git a/cmswww/inventory.go b/cmswww/inventory.go index e8d9c42..3f637dd 100644 --- a/cmswww/inventory.go +++ b/cmswww/inventory.go @@ -19,18 +19,6 @@ type inventoryRecord struct { payments []BackendInvoiceMDPayment // payments metadata } -// updateInventoryRecord updates an existing Politea record within the database. -// -// This function must be called WITH the mutex held. -func (c *cmswww) updateInventoryRecord(record pd.Record) error { - dbInvoice, err := c.convertRecordToDatabaseInvoice(record) - if err != nil { - return err - } - - return c.db.UpdateInvoice(dbInvoice) -} - // newInventoryRecord adds a Politeia record to the database. // // This function must be called WITH the mutex held. diff --git a/cmswww/invoice.go b/cmswww/invoice.go index 35a1394..aa7b2cd 100644 --- a/cmswww/invoice.go +++ b/cmswww/invoice.go @@ -283,6 +283,64 @@ func (c *cmswww) updateMDPayments( return util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) } +func (c *cmswww) addMDChange( + invoiceToken string, + ts int64, + status v1.InvoiceStatusT, + adminPublicKey string, + reason *string, +) (*BackendInvoiceMDChange, error) { + // Create the change record. + mdChange := BackendInvoiceMDChange{ + Version: VersionBackendInvoiceMDChange, + Timestamp: time.Now().Unix(), + NewStatus: status, + Reason: reason, + } + + blob, err := json.Marshal(mdChange) + if err != nil { + return nil, err + } + + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + + pdCommand := pd.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: invoiceToken, + MDAppend: []pd.MetadataStream{ + { + ID: mdStreamChanges, + Payload: string(blob), + }, + }, + } + + responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, + pdCommand) + if err != nil { + return nil, err + } + + var pdReply pd.UpdateVettedMetadataReply + err = json.Unmarshal(responseBody, &pdReply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "UpdateVettedMetadataReply: %v", err) + } + + // Verify the challenge. + err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) + if err != nil { + return nil, err + } + + return &mdChange, nil +} + func (c *cmswww) createInvoicePayment( dbInvoice *database.Invoice, usdDCRRate float64, @@ -390,6 +448,7 @@ func (c *cmswww) updateInvoicePayment( } } + // Update the Politeia record. ts := time.Now().Unix() err := c.updateMDPayments(dbInvoice, true, ts) if err != nil { @@ -700,74 +759,25 @@ func (c *cmswww) HandleSetInvoiceStatus( return nil, err } - // Create the change record. - changes := BackendInvoiceMDChange{ - Version: VersionBackendInvoiceMDChange, - Timestamp: time.Now().Unix(), - NewStatus: sis.Status, - Reason: sis.Reason, - } - - var ok bool - changes.AdminPublicKey, ok = database.ActiveIdentityString(user.Identities) + adminPublicKey, ok := database.ActiveIdentityString(user.Identities) if !ok { return nil, fmt.Errorf("invalid admin identity: %v", user.ID) } - blob, err := json.Marshal(changes) - if err != nil { - return nil, err - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - pdCommand := pd.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: sis.Token, - MDAppend: []pd.MetadataStream{ - { - ID: mdStreamChanges, - Payload: string(blob), - }, - }, - } - - responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, - pdCommand) + mdChange, err := c.addMDChange(sis.Token, time.Now().Unix(), sis.Status, + adminPublicKey, sis.Reason) if err != nil { return nil, err } - var pdReply pd.UpdateVettedMetadataReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return nil, fmt.Errorf("Could not unmarshal UpdateVettedMetadataReply: %v", - err) - } + // Update the database with the metadata changes. + dbInvoiceChange := convertStreamChangeToDatabaseInvoiceChange(*mdChange) + dbInvoice.Changes = append(dbInvoice.Changes, dbInvoiceChange) - // Verify the challenge. - err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return nil, err - } + dbInvoice.Status = dbInvoiceChange.NewStatus + dbInvoice.StatusChangeReason = dbInvoiceChange.Reason - // Update the database with the metadata changes. - dbInvoice.Changes = append(dbInvoice.Changes, database.InvoiceChange{ - Timestamp: changes.Timestamp, - AdminPublicKey: changes.AdminPublicKey, - NewStatus: changes.NewStatus, - }) - dbInvoice.Status = changes.NewStatus - if changes.Reason != nil { - dbInvoice.Changes[len(dbInvoice.Changes)-1].Reason = *changes.Reason - dbInvoice.StatusChangeReason = *changes.Reason - } else { - dbInvoice.StatusChangeReason = "" - } err = c.db.UpdateInvoice(dbInvoice) if err != nil { return nil, err @@ -854,7 +864,7 @@ func (c *cmswww) HandleInvoiceDetails( invoice.Files = convertRecordFilesToInvoiceFiles(pdReply.Record.Files) // Update the database with the files fetched from Politeiad. - err = c.db.CreateInvoiceFiles(dbInvoice.Token, + err = c.db.CreateInvoiceFiles(dbInvoice.Token, dbInvoice.Version, convertRecordFilesToDatabaseInvoiceFiles(pdReply.Record.Files)) if err != nil { return nil, err @@ -1036,7 +1046,7 @@ func (c *cmswww) HandleEditInvoice( } // Create the change record. - changes, err := json.Marshal(BackendInvoiceMDChange{ + mdChanges, err := json.Marshal(BackendInvoiceMDChange{ Version: VersionBackendInvoiceMDChange, Timestamp: ts, NewStatus: v1.InvoiceStatusUnreviewedChanges, @@ -1045,6 +1055,21 @@ func (c *cmswww) HandleEditInvoice( return nil, err } + /* + var delFiles []string + for _, v := range invRecord.record.Files { + found := false + for _, c := range ep.Files { + if v.Name == c.Name { + found = true + } + } + if !found { + delFiles = append(delFiles, v.Name) + } + } + */ + u := pd.UpdateRecord{ Token: ei.Token, Challenge: hex.EncodeToString(challenge), @@ -1054,7 +1079,7 @@ func (c *cmswww) HandleEditInvoice( }}, MDAppend: []pd.MetadataStream{{ ID: mdStreamChanges, - Payload: string(changes), + Payload: string(mdChanges), }}, FilesAdd: convertInvoiceFilesToRecordFiles(ei.Files), } @@ -1078,14 +1103,13 @@ func (c *cmswww) HandleEditInvoice( return nil, err } - // Update the database with the metadata changes. - dbInvoice.Changes = append(dbInvoice.Changes, database.InvoiceChange{ - Timestamp: ts, - NewStatus: v1.InvoiceStatusUnreviewedChanges, - }) - dbInvoice.Version = pdUpdateRecordReply.Record.Version - dbInvoice.Status = v1.InvoiceStatusUnreviewedChanges - err = c.db.UpdateInvoice(dbInvoice) + dbInvoice, err = c.convertRecordToDatabaseInvoice( + pdUpdateRecordReply.Record) + if err != nil { + return nil, err + } + + err = c.db.CreateInvoice(dbInvoice) if err != nil { return nil, err } From 823ec23b607b99892bb6b98a7638254da0d59bc8 Mon Sep 17 00:00:00 2001 From: Sean Durkin Date: Fri, 1 Feb 2019 00:51:53 -0500 Subject: [PATCH 4/5] Refactor invoice functions and fix issues with cockroachdb tables. --- cmswww/cmd/cmswwwcli/main.go | 4 +- cmswww/cmd/cmswwwdataload/main.go | 3 +- cmswww/database/cockroachdb/cockroachdb.go | 82 ++- cmswww/database/cockroachdb/encoding.go | 15 +- cmswww/database/cockroachdb/models.go | 28 +- cmswww/database/database.go | 31 +- cmswww/invoice.go | 734 +-------------------- cmswww/invoice_admin.go | 591 +++++++++++++++++ cmswww/invoice_util.go | 183 +++++ cmswww/payment.go | 3 +- 10 files changed, 889 insertions(+), 785 deletions(-) create mode 100644 cmswww/invoice_admin.go create mode 100644 cmswww/invoice_util.go diff --git a/cmswww/cmd/cmswwwcli/main.go b/cmswww/cmd/cmswwwcli/main.go index b4ce689..e9cbcec 100644 --- a/cmswww/cmd/cmswwwcli/main.go +++ b/cmswww/cmd/cmswwwcli/main.go @@ -47,7 +47,9 @@ func _main() error { func main() { err := _main() if err != nil { - //fmt.Fprintf(os.Stderr, "%v\n", err) + if config.Verbose { + fmt.Fprintf(os.Stderr, "cli error: %v\n", err) + } os.Exit(1) } } diff --git a/cmswww/cmd/cmswwwdataload/main.go b/cmswww/cmd/cmswwwdataload/main.go index a900288..e1ed11a 100644 --- a/cmswww/cmd/cmswwwdataload/main.go +++ b/cmswww/cmd/cmswwwdataload/main.go @@ -34,6 +34,7 @@ var ( ) func createLogFile(path string) (*os.File, error) { + os.Remove(path) return os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) } @@ -626,7 +627,7 @@ func _main() error { func main() { err := _main() if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + fmt.Fprintf(os.Stderr, "main error: %v\n", err) } stopServers() if err != nil { diff --git a/cmswww/database/cockroachdb/cockroachdb.go b/cmswww/database/cockroachdb/cockroachdb.go index 8523fda..1007744 100644 --- a/cmswww/database/cockroachdb/cockroachdb.go +++ b/cmswww/database/cockroachdb/cockroachdb.go @@ -45,6 +45,18 @@ func (c *cockroachdb) addWhereClause(db *gorm.DB, paramsMap map[string]interface return db } +func (c *cockroachdb) fetchInvoiceFiles(invoice *Invoice) error { + result := c.db.Where("invoice_token = ? and invoice_version = ?", + invoice.Token, invoice.Version).Find(&invoice.Files) + return result.Error +} + +func (c *cockroachdb) fetchInvoicePayments(invoice *Invoice) error { + result := c.db.Where("invoice_token = ? and invoice_version = ?", + invoice.Token, invoice.Version).Find(&invoice.Payments) + return result.Error +} + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" func (c *cockroachdb) dropTable(tableName string) error { @@ -221,19 +233,19 @@ func (c *cockroachdb) GetUsers(username string, page int) ([]database.User, int, return dbUsers, numMatches, nil } -// Create new invoice. +// Create a new invoice version. // // CreateInvoice satisfies the backend interface. func (c *cockroachdb) CreateInvoice(dbInvoice *database.Invoice) error { invoice := EncodeInvoice(dbInvoice) - log.Debugf("CreateInvoice: %v", invoice.Token) + log.Debugf("CreateInvoice: %v %v", invoice.Token, len(dbInvoice.Files)) return c.db.Create(invoice).Error } -// Update existing invoice. +// Update an existing invoice version. // -// CreateInvoice satisfies the backend interface. +// UpdateInvoice satisfies the backend interface. func (c *cockroachdb) UpdateInvoice(dbInvoice *database.Invoice) error { invoice := EncodeInvoice(dbInvoice) @@ -242,7 +254,9 @@ func (c *cockroachdb) UpdateInvoice(dbInvoice *database.Invoice) error { return c.db.Save(invoice).Error } -// Return invoice by its token. +// Return the latest invoice version given its token. +// +// GetInvoiceByToken satisfies the backend interface. func (c *cockroachdb) GetInvoiceByToken(token string) (*database.Invoice, error) { log.Debugf("GetInvoiceByToken: %v", token) @@ -268,22 +282,22 @@ func (c *cockroachdb) GetInvoiceByToken(token string) (*database.Invoice, error) return nil, result.Error } - result = c.db.Where("invoice_token = ?", invoice.Token).Find( - &invoice.Files) - if result.Error != nil { - return nil, result.Error + err := c.fetchInvoiceFiles(&invoice) + if err != nil { + return nil, err } - result = c.db.Where("invoice_token = ?", invoice.Token).Find( - &invoice.Payments) - if result.Error != nil { - return nil, result.Error + err = c.fetchInvoicePayments(&invoice) + if err != nil { + return nil, err } return DecodeInvoice(&invoice) } -// Return a list of invoices. +// Return a list of the latest invoices. +// +// GetInvoices satisfies the backend interface. func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]database.Invoice, int, error) { log.Debugf("GetInvoices") @@ -343,6 +357,25 @@ func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]d return nil, 0, result.Error } + // Set the invoice payments and files on each invoice. + if invoicesRequest.IncludePayments || invoicesRequest.IncludeFiles { + for idx := range invoices { + if invoicesRequest.IncludeFiles { + err = c.fetchInvoiceFiles(&invoices[idx]) + if err != nil { + return nil, 0, err + } + } + + if invoicesRequest.IncludePayments { + err = c.fetchInvoicePayments(&invoices[idx]) + if err != nil { + return nil, 0, err + } + } + } + } + // If the number of users returned equals the page size, // find the count of all users that match the query. numMatches := len(invoices) @@ -362,23 +395,40 @@ func (c *cockroachdb) GetInvoices(invoicesRequest database.InvoicesRequest) ([]d return dbInvoices, numMatches, nil } -func (c *cockroachdb) UpdateInvoicePayment(dbInvoicePayment *database.InvoicePayment) error { +// Update an existing invoice's payment. +// +// UpdateInvoicePayment satisfies the backend interface. +func (c *cockroachdb) UpdateInvoicePayment( + token, version string, + dbInvoicePayment *database.InvoicePayment, +) error { invoicePayment := EncodeInvoicePayment(dbInvoicePayment) + invoicePayment.InvoiceToken = token + invoicePayment.InvoiceVersion = version log.Debugf("UpdateInvoicePayment: %v", invoicePayment.InvoiceToken) return c.db.Save(invoicePayment).Error } +// Create files for an invoice version. +// +// CreateInvoiceFiles satisfies the backend interface. func (c *cockroachdb) CreateInvoiceFiles( token, version string, dbInvoiceFiles []database.InvoiceFile, ) error { log.Debugf("CreateInvoiceFiles: %v", token) - for _, dbInvoiceFile := range dbInvoiceFiles { + for idx, dbInvoiceFile := range dbInvoiceFiles { invoiceFile := EncodeInvoiceFile(&dbInvoiceFile) + + // Start the ID at 1 because gorm thinks it's a blank field if 0 is + // passed and will automatically derive a value for it. + invoiceFile.ID = int64(idx + 1) + invoiceFile.InvoiceToken = token + invoiceFile.InvoiceVersion = version err := c.db.Create(invoiceFile).Error if err != nil { diff --git a/cmswww/database/cockroachdb/encoding.go b/cmswww/database/cockroachdb/encoding.go index 98ae35e..16b000d 100644 --- a/cmswww/database/cockroachdb/encoding.go +++ b/cmswww/database/cockroachdb/encoding.go @@ -218,15 +218,22 @@ func EncodeInvoice(dbInvoice *database.Invoice) *Invoice { invoice.Proposal = dbInvoice.Proposal invoice.Version = dbInvoice.Version - for _, dbInvoiceFile := range dbInvoice.Files { + for idx, dbInvoiceFile := range dbInvoice.Files { invoiceFile := EncodeInvoiceFile(&dbInvoiceFile) + + // Start the ID at 1 because gorm thinks it's a blank field if 0 is + // passed and will automatically derive a value for it. + invoiceFile.ID = int64(idx + 1) + invoiceFile.InvoiceToken = invoice.Token + invoiceFile.InvoiceVersion = invoice.Version invoice.Files = append(invoice.Files, *invoiceFile) } for _, dbInvoiceChange := range dbInvoice.Changes { invoiceChange := EncodeInvoiceChange(&dbInvoiceChange) invoiceChange.InvoiceToken = invoice.Token + invoiceChange.InvoiceVersion = invoice.Version invoice.Changes = append(invoice.Changes, *invoiceChange) invoice.Status = invoiceChange.NewStatus } @@ -234,6 +241,7 @@ func EncodeInvoice(dbInvoice *database.Invoice) *Invoice { for _, dbInvoicePayment := range dbInvoice.Payments { invoicePayment := EncodeInvoicePayment(&dbInvoicePayment) invoicePayment.InvoiceToken = invoice.Token + invoicePayment.InvoiceVersion = invoice.Version invoice.Payments = append(invoice.Payments, *invoicePayment) } @@ -270,8 +278,6 @@ func EncodeInvoiceChange(dbInvoiceChange *database.InvoiceChange) *InvoiceChange func EncodeInvoicePayment(dbInvoicePayment *database.InvoicePayment) *InvoicePayment { invoicePayment := InvoicePayment{} - invoicePayment.ID = uint(dbInvoicePayment.ID) - //invoicePayment.InvoiceToken = dbInvoicePayment.InvoiceToken invoicePayment.IsTotalCost = dbInvoicePayment.IsTotalCost invoicePayment.Address = dbInvoicePayment.Address invoicePayment.Amount = uint(dbInvoicePayment.Amount) @@ -349,8 +355,7 @@ func DecodeInvoiceChange(invoiceChange *InvoiceChange) *database.InvoiceChange { func DecodeInvoicePayment(invoicePayment *InvoicePayment) *database.InvoicePayment { dbInvoicePayment := database.InvoicePayment{} - dbInvoicePayment.ID = uint64(invoicePayment.ID) - //dbInvoicePayment.InvoiceToken = invoicePayment.InvoiceToken + //dbInvoicePayment.ID = uint64(invoicePayment.ID) dbInvoicePayment.IsTotalCost = invoicePayment.IsTotalCost dbInvoicePayment.Address = invoicePayment.Address dbInvoicePayment.Amount = uint64(invoicePayment.Amount) diff --git a/cmswww/database/cockroachdb/models.go b/cmswww/database/cockroachdb/models.go index 77fbb87..5878c8c 100644 --- a/cmswww/database/cockroachdb/models.go +++ b/cmswww/database/cockroachdb/models.go @@ -76,9 +76,9 @@ type Invoice struct { MerkleRoot string `gorm:"not_null"` Proposal string - Files []InvoiceFile - Changes []InvoiceChange - Payments []InvoicePayment + Files []InvoiceFile `gorm:"foreignkey:invoice_token,invoice_version;association_foreignkey:token,version"` + Changes []InvoiceChange `gorm:"foreignkey:invoice_token,invoice_version;association_foreignkey:token,version"` + Payments []InvoicePayment `gorm:"foreignkey:invoice_token,invoice_version;association_foreignkey:token,version"` // gorm.Model fields, included manually CreatedAt time.Time @@ -91,12 +91,12 @@ func (i Invoice) TableName() string { } type InvoiceFile struct { - gorm.Model - InvoiceToken string - InvoiceVersion string - Name string - MIME string - Digest string + ID int64 `gorm:"primary_key;auto_increment:false"` + InvoiceToken string `gorm:"primary_key"` + InvoiceVersion string `gorm:"primary_key"` + Name string `gorm:"not_null"` + MIME string `gorm:"not_null"` + Digest string `gorm:"not_null"` Payload string `gorm:"type:text"` } @@ -105,9 +105,8 @@ func (i InvoiceFile) TableName() string { } type InvoiceChange struct { - gorm.Model - InvoiceToken string - InvoiceVersion string + InvoiceToken string `gorm:"primary_key"` + InvoiceVersion string `gorm:"primary_key"` AdminPublicKey string NewStatus uint Timestamp time.Time @@ -118,9 +117,8 @@ func (i InvoiceChange) TableName() string { } type InvoicePayment struct { - gorm.Model - InvoiceToken string - InvoiceVersion string + InvoiceToken string `gorm:"primary_key"` + InvoiceVersion string `gorm:"primary_key"` IsTotalCost bool `gorm:"not_null"` Address string `gorm:"not_null"` Amount uint `gorm:"not_null"` diff --git a/cmswww/database/database.go b/cmswww/database/database.go index dcd8b51..a85c997 100644 --- a/cmswww/database/database.go +++ b/cmswww/database/database.go @@ -31,11 +31,13 @@ var ( // InvoicesRequest is used for passing parameters into the // GetInvoices() function. type InvoicesRequest struct { - UserID string - Month uint16 - Year uint16 - StatusMap map[v1.InvoiceStatusT]bool - Page int + UserID string + Month uint16 + Year uint16 + StatusMap map[v1.InvoiceStatusT]bool + IncludePayments bool + IncludeFiles bool + Page int } // Database interface that is required by the web server. @@ -51,12 +53,12 @@ type Database interface { GetUsers(username string, page int) ([]User, int, error) // Returns a list of users and total count that match the provided username. // Invoice functions - CreateInvoice(*Invoice) error // Create new invoice - UpdateInvoice(*Invoice) error // Update existing invoice - GetInvoiceByToken(string) (*Invoice, error) // Return invoice given its token - GetInvoices(InvoicesRequest) ([]Invoice, int, error) // Return a list of invoices - UpdateInvoicePayment(*InvoicePayment) error // Update an existing invoice's payment - CreateInvoiceFiles(string, string, []InvoiceFile) error // Create invoice files + CreateInvoice(*Invoice) error // Create a new invoice version + UpdateInvoice(*Invoice) error // Update an existing invoice version + GetInvoiceByToken(string) (*Invoice, error) // Return the latest invoice version given its token + GetInvoices(InvoicesRequest) ([]Invoice, int, error) // Return a list of the latest invoices + UpdateInvoicePayment(string, string, *InvoicePayment) error // Update an existing invoice's payment + CreateInvoiceFiles(string, string, []InvoiceFile) error // Create files for an invoice version. DeleteAllData() error // Delete all data from all tables @@ -100,7 +102,11 @@ type Identity struct { Deactivated int64 } -// Invoice represents an invoice submitted by a contractor for payment. +// Invoice represents a specific "invoice version" submitted by a contractor +// for payment. Invoices have a unique token, and may have multiple versions +// as contractors can submit edits to them. So the `Invoice` struct represents +// a specific version of an invoice, thus it has 2 primary keys: `Token` and +// `Version`. type Invoice struct { Token string // Unique id for this invoice Version string // Version number of this invoice @@ -137,7 +143,6 @@ type InvoiceChange struct { } type InvoicePayment struct { - ID uint64 IsTotalCost bool // Whether this payment represents the total cost of the invoice Address string Amount uint64 diff --git a/cmswww/invoice.go b/cmswww/invoice.go index aa7b2cd..0b6456a 100644 --- a/cmswww/invoice.go +++ b/cmswww/invoice.go @@ -1,17 +1,13 @@ package main import ( - "encoding/base64" - "encoding/csv" "encoding/hex" "encoding/json" "fmt" "net/http" "strconv" - "strings" "time" - "github.com/decred/dcrd/dcrutil" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/util" @@ -19,64 +15,6 @@ import ( "github.com/decred/contractor-mgmt/cmswww/database" ) -var ( - validStatusTransitions = map[v1.InvoiceStatusT][]v1.InvoiceStatusT{ - v1.InvoiceStatusNotReviewed: { - v1.InvoiceStatusApproved, - v1.InvoiceStatusRejected, - }, - v1.InvoiceStatusRejected: { - v1.InvoiceStatusApproved, - v1.InvoiceStatusUnreviewedChanges, - }, - v1.InvoiceStatusUnreviewedChanges: { - v1.InvoiceStatusApproved, - v1.InvoiceStatusRejected, - }, - v1.InvoiceStatusApproved: { - v1.InvoiceStatusPaid, - }, - } -) - -func statusInSlice(arr []v1.InvoiceStatusT, status v1.InvoiceStatusT) bool { - for _, s := range arr { - if status == s { - return true - } - } - - return false -} - -func validateStatusTransition( - dbInvoice *database.Invoice, - newStatus v1.InvoiceStatusT, - reason *string, -) error { - validStatuses, ok := validStatusTransitions[dbInvoice.Status] - if !ok { - log.Errorf("status not supported: %v", dbInvoice.Status) - return v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInvoiceStatusTransition, - } - } - - if !statusInSlice(validStatuses, newStatus) { - return v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInvoiceStatusTransition, - } - } - - if newStatus == v1.InvoiceStatusRejected && reason == nil { - return v1.UserError{ - ErrorCode: v1.ErrorStatusReasonNotProvided, - } - } - - return nil -} - func (c *cmswww) validateInvoiceUnique(user *database.User, month, year uint16) error { invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ UserID: strconv.FormatUint(user.ID, 10), @@ -97,607 +35,6 @@ func (c *cmswww) validateInvoiceUnique(user *database.User, month, year uint16) return nil } -func (c *cmswww) refreshExistingInvoicePayments(dbInvoice *database.Invoice) error { - for _, dbInvoicePayment := range dbInvoice.Payments { - if dbInvoicePayment.TxID != "" { - continue - } - - dbInvoicePayment.PollExpiry = - time.Now().Add(pollExpiryDuration).Unix() - - err := c.db.UpdateInvoicePayment(&dbInvoicePayment) - if err != nil { - return err - } - - c.addInvoicePaymentForPolling(dbInvoice.Token, &dbInvoicePayment) - } - - return nil -} - -func (c *cmswww) deriveTotalCostFromInvoice( - dbInvoice *database.Invoice, - invoicePayment *v1.InvoicePayment, -) error { - b, err := base64.StdEncoding.DecodeString(dbInvoice.Files[0].Payload) - if err != nil { - return err - } - - csvReader := csv.NewReader(strings.NewReader(string(b))) - csvReader.Comma = v1.PolicyInvoiceFieldDelimiterChar - csvReader.Comment = v1.PolicyInvoiceCommentChar - csvReader.TrimLeadingSpace = true - - records, err := csvReader.ReadAll() - if err != nil { - return err - } - - for _, record := range records { - for idx := range v1.InvoiceFields { - switch idx { - case 4: - hours, err := strconv.ParseUint(record[idx], 10, 64) - if err != nil { - return err - } - - invoicePayment.TotalHours += hours - case 5: - totalCost, err := strconv.ParseUint(record[idx], 10, 64) - if err != nil { - return err - } - - invoicePayment.TotalCostUSD += totalCost - } - } - } - - return nil -} - -func (c *cmswww) createInvoiceReview(invoice *database.Invoice) (*v1.InvoiceReview, error) { - invoiceReview := v1.InvoiceReview{ - UserID: strconv.FormatUint(invoice.UserID, 10), - Username: invoice.Username, - Token: invoice.Token, - LineItems: make([]v1.InvoiceReviewLineItem, 0), - } - - b, err := base64.StdEncoding.DecodeString(invoice.Files[0].Payload) - if err != nil { - return nil, err - } - - csvReader := csv.NewReader(strings.NewReader(string(b))) - csvReader.Comma = v1.PolicyInvoiceFieldDelimiterChar - csvReader.Comment = v1.PolicyInvoiceCommentChar - csvReader.TrimLeadingSpace = true - - records, err := csvReader.ReadAll() - if err != nil { - return nil, err - } - - for _, record := range records { - lineItem := v1.InvoiceReviewLineItem{} - for idx := range v1.InvoiceFields { - var err error - switch idx { - case 0: - lineItem.Type = record[idx] - case 1: - lineItem.Subtype = record[idx] - case 2: - lineItem.Description = record[idx] - case 3: - lineItem.Proposal = record[idx] - case 4: - lineItem.Hours, err = strconv.ParseUint(record[idx], 10, 64) - if err != nil { - return nil, err - } - - invoiceReview.TotalHours += lineItem.Hours - case 5: - lineItem.TotalCost, err = strconv.ParseUint(record[idx], 10, 64) - if err != nil { - return nil, err - } - - invoiceReview.TotalCostUSD += lineItem.TotalCost - } - } - - invoiceReview.LineItems = append(invoiceReview.LineItems, lineItem) - } - - return &invoiceReview, nil -} - -func (c *cmswww) updateMDPayments( - dbInvoice *database.Invoice, - updatingInvoicePayment bool, - ts int64, -) error { - // Create the payments metadata record. - mdPayments, err := convertDatabaseInvoicePaymentsToStreamPayments(dbInvoice) - if err != nil { - return err - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return fmt.Errorf("could not create challenge: %v", err) - } - - pdCommand := pd.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: dbInvoice.Token, - MDOverwrite: []pd.MetadataStream{ - { - ID: mdStreamPayments, - Payload: mdPayments, - }, - }, - } - - // Create the change metadata record if an existing invoice payment - // is being updated. - if updatingInvoicePayment && dbInvoice.Status != v1.InvoiceStatusPaid { - mdChange, err := json.Marshal(BackendInvoiceMDChange{ - Version: VersionBackendInvoiceMDChange, - Timestamp: ts, - NewStatus: v1.InvoiceStatusPaid, - }) - if err != nil { - return fmt.Errorf("cannot marshal backend change: %v", err) - } - - pdCommand.MDAppend = []pd.MetadataStream{ - { - ID: mdStreamChanges, - Payload: string(mdChange), - }, - } - } - - responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, - pdCommand) - if err != nil { - return err - } - - var pdReply pd.UpdateVettedMetadataReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return fmt.Errorf("Could not unmarshal UpdateVettedMetadataReply: %v", - err) - } - - // Verify the challenge. - return util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) -} - -func (c *cmswww) addMDChange( - invoiceToken string, - ts int64, - status v1.InvoiceStatusT, - adminPublicKey string, - reason *string, -) (*BackendInvoiceMDChange, error) { - // Create the change record. - mdChange := BackendInvoiceMDChange{ - Version: VersionBackendInvoiceMDChange, - Timestamp: time.Now().Unix(), - NewStatus: status, - Reason: reason, - } - - blob, err := json.Marshal(mdChange) - if err != nil { - return nil, err - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - pdCommand := pd.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: invoiceToken, - MDAppend: []pd.MetadataStream{ - { - ID: mdStreamChanges, - Payload: string(blob), - }, - }, - } - - responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, - pdCommand) - if err != nil { - return nil, err - } - - var pdReply pd.UpdateVettedMetadataReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "UpdateVettedMetadataReply: %v", err) - } - - // Verify the challenge. - err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return nil, err - } - - return &mdChange, nil -} - -func (c *cmswww) createInvoicePayment( - dbInvoice *database.Invoice, - usdDCRRate float64, - costUSD uint64, -) (*v1.InvoicePayment, error) { - invoicePayment := v1.InvoicePayment{ - UserID: strconv.FormatUint(dbInvoice.UserID, 10), - Username: dbInvoice.Username, - Token: dbInvoice.Token, - } - - var recreatingTotalCostPayment bool - dbInvoicePayment := &database.InvoicePayment{} - if costUSD == 0 { - err := c.deriveTotalCostFromInvoice(dbInvoice, &invoicePayment) - if err != nil { - return nil, err - } - - // If there's already payments on this invoice, determine - // if one of them is for the total cost. - for i, payment := range dbInvoice.Payments { - if payment.IsTotalCost { - recreatingTotalCostPayment = true - dbInvoicePayment = &dbInvoice.Payments[i] - break - } - } - - dbInvoicePayment.IsTotalCost = true - } else { - invoicePayment.TotalCostUSD = costUSD - } - - invoicePayment.TotalCostDCR = float64(invoicePayment.TotalCostUSD) / usdDCRRate - - // Generate the user's address. - user, err := c.db.GetUserById(dbInvoice.UserID) - if err != nil { - return nil, err - } - - // Create or update the invoice payment in the DB. - address, txNotBefore, err := c.derivePaymentInfo(user) - if err != nil { - return nil, err - } - - amount, err := dcrutil.NewAmount(invoicePayment.TotalCostDCR) - if err != nil { - return nil, err - } - - oldAddress := dbInvoicePayment.Address - - dbInvoicePayment.Address = address - dbInvoicePayment.TxNotBefore = txNotBefore - dbInvoicePayment.Amount = uint64(amount) - dbInvoicePayment.PollExpiry = time.Now().Add(pollExpiryDuration).Unix() - if !recreatingTotalCostPayment { - dbInvoice.Payments = append(dbInvoice.Payments, *dbInvoicePayment) - } - - err = c.updateMDPayments(dbInvoice, false, 0) - if err != nil { - return nil, err - } - - if recreatingTotalCostPayment { - err = c.db.UpdateInvoicePayment(dbInvoicePayment) - } else { - err = c.db.UpdateInvoice(dbInvoice) - } - if err != nil { - return nil, err - } - - if recreatingTotalCostPayment { - c.removeInvoicePaymentsFromPolling([]string{oldAddress}) - } - c.addInvoicePaymentForPolling(dbInvoice.Token, dbInvoicePayment) - - invoicePayment.PaymentAddress = address - return &invoicePayment, nil -} - -func (c *cmswww) updateInvoicePayment( - dbInvoice *database.Invoice, - address string, - amount uint64, - txID string, -) error { - var dbInvoicePayment *database.InvoicePayment - for idx, payment := range dbInvoice.Payments { - if payment.Amount == amount && payment.Address == address { - dbInvoice.Payments[idx].TxID = txID - dbInvoicePayment = &payment - break - } - } - - if dbInvoicePayment == nil { - return v1.UserError{ - ErrorCode: v1.ErrorStatusInvoicePaymentNotFound, - } - } - - // Update the Politeia record. - ts := time.Now().Unix() - err := c.updateMDPayments(dbInvoice, true, ts) - if err != nil { - return err - } - - // Update the invoice in the database. - if dbInvoice.Status != v1.InvoiceStatusPaid { - // Update the status in the database if necessary. - dbInvoice.Status = v1.InvoiceStatusPaid - dbInvoice.Changes = append(dbInvoice.Changes, database.InvoiceChange{ - Timestamp: ts, - NewStatus: v1.InvoiceStatusPaid, - }) - } - err = c.db.UpdateInvoice(dbInvoice) - if err != nil { - return fmt.Errorf("cannot update invoice with token %v: %v", - dbInvoice.Token, err) - } - - return nil -} - -func (c *cmswww) fetchInvoiceFileIfNecessary(invoice *database.Invoice) error { - if len(invoice.Files) == 0 { - return nil - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return err - } - - responseBody, err := c.rpc(http.MethodPost, pd.GetVettedRoute, - pd.GetVetted{ - Token: invoice.Token, - Challenge: hex.EncodeToString(challenge), - }) - if err != nil { - return err - } - - var pdReply pd.GetVettedReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return fmt.Errorf("Could not unmarshal "+ - "GetVettedReply: %v", err) - } - - // Verify the challenge. - err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return err - } - - invoice.Files = convertRecordFilesToDatabaseInvoiceFiles(pdReply.Record.Files) - return nil -} - -// HandleInvoices returns an array of all invoices. -func (c *cmswww) HandleInvoices( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - i := req.(*v1.Invoices) - - statusMap := make(map[v1.InvoiceStatusT]bool) - if i.Status != v1.InvoiceStatusInvalid { - statusMap[i.Status] = true - } - - invoices, numMatches, err := c.getInvoices(database.InvoicesRequest{ - Month: i.Month, - Year: i.Year, - StatusMap: statusMap, - Page: int(i.Page), - }) - if err != nil { - return nil, err - } - - return &v1.InvoicesReply{ - Invoices: invoices, - TotalMatches: uint64(numMatches), - }, nil -} - -// HandleReviewInvoices returns a list of all unreviewed invoices. -func (c *cmswww) HandleReviewInvoices( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - ri := req.(*v1.ReviewInvoices) - - invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ - Month: ri.Month, - Year: ri.Year, - StatusMap: map[v1.InvoiceStatusT]bool{ - v1.InvoiceStatusNotReviewed: true, - v1.InvoiceStatusUnreviewedChanges: true, - }, - Page: -1, - }) - if err != nil { - return nil, err - } - - var invoiceReviews []v1.InvoiceReview - - for _, invoice := range invoices { - err := c.fetchInvoiceFileIfNecessary(&invoice) - if err != nil { - return nil, err - } - - invoiceReview, err := c.createInvoiceReview(&invoice) - if err != nil { - return nil, err - } - - invoiceReviews = append(invoiceReviews, *invoiceReview) - } - - return &v1.ReviewInvoicesReply{ - Invoices: invoiceReviews, - }, nil -} - -// HandlePayInvoices creates new invoice payments and returns their data. -func (c *cmswww) HandlePayInvoices( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - pi := req.(*v1.PayInvoices) - - invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ - Month: pi.Month, - Year: pi.Year, - StatusMap: map[v1.InvoiceStatusT]bool{ - v1.InvoiceStatusApproved: true, - }, - Page: -1, - }) - if err != nil { - return nil, err - } - - invoicePayments := make([]v1.InvoicePayment, 0) - - for _, inv := range invoices { - invoice, err := c.db.GetInvoiceByToken(inv.Token) - if err != nil { - return nil, err - } - - err = c.fetchInvoiceFileIfNecessary(invoice) - if err != nil { - return nil, err - } - - err = c.refreshExistingInvoicePayments(invoice) - if err != nil { - return nil, err - } - - invoicePayment, err := c.createInvoicePayment(invoice, pi.USDDCRRate, 0) - if err != nil { - return nil, err - } - - invoicePayments = append(invoicePayments, *invoicePayment) - } - - return &v1.PayInvoicesReply{ - Invoices: invoicePayments, - }, nil -} - -// HandlePayInvoice creates a new invoice payment and returns it. -func (c *cmswww) HandlePayInvoice( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - pi := req.(*v1.PayInvoice) - - invoice, err := c.db.GetInvoiceByToken(pi.Token) - if err != nil { - return nil, err - } - - err = c.fetchInvoiceFileIfNecessary(invoice) - if err != nil { - return nil, err - } - - err = c.refreshExistingInvoicePayments(invoice) - if err != nil { - return nil, err - } - - invoicePayment, err := c.createInvoicePayment(invoice, pi.USDDCRRate, - pi.CostUSD) - if err != nil { - return nil, err - } - - return &v1.PayInvoiceReply{ - Invoice: *invoicePayment, - }, nil -} - -// HandleUpdateInvoicePayment updates a payment for an invoice. -func (c *cmswww) HandleUpdateInvoicePayment( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - aip := req.(*v1.UpdateInvoicePayment) - - dbInvoice, err := c.db.GetInvoiceByToken(aip.Token) - if err != nil { - if err == database.ErrInvoiceNotFound { - return nil, v1.UserError{ - ErrorCode: v1.ErrorStatusInvoiceNotFound, - } - } - - return nil, err - } - - err = c.updateInvoicePayment(dbInvoice, aip.Address, uint64(aip.Amount), - aip.TxID) - if err != nil { - return nil, err - } - - return &v1.UpdateInvoicePaymentReply{}, nil -} - // HandleUserInvoices returns an array of user's invoices. func (c *cmswww) HandleUserInvoices( req interface{}, @@ -727,76 +64,6 @@ func (c *cmswww) HandleUserInvoices( }, nil } -// HandleSetInvoiceStatus changes the status of an existing invoice -// from unreviewed to either published or rejected. -func (c *cmswww) HandleSetInvoiceStatus( - req interface{}, - user *database.User, - w http.ResponseWriter, - r *http.Request, -) (interface{}, error) { - sis := req.(*v1.SetInvoiceStatus) - - err := checkPublicKeyAndSignature(user, sis.PublicKey, sis.Signature, - sis.Token, strconv.FormatUint(uint64(sis.Status), 10)) - if err != nil { - return nil, err - } - - dbInvoice, err := c.db.GetInvoiceByToken(sis.Token) - if err != nil { - if err == database.ErrInvoiceNotFound { - return nil, v1.UserError{ - ErrorCode: v1.ErrorStatusInvoiceNotFound, - } - } - - return nil, err - } - - err = validateStatusTransition(dbInvoice, sis.Status, sis.Reason) - if err != nil { - return nil, err - } - - adminPublicKey, ok := database.ActiveIdentityString(user.Identities) - if !ok { - return nil, fmt.Errorf("invalid admin identity: %v", - user.ID) - } - - mdChange, err := c.addMDChange(sis.Token, time.Now().Unix(), sis.Status, - adminPublicKey, sis.Reason) - if err != nil { - return nil, err - } - - // Update the database with the metadata changes. - dbInvoiceChange := convertStreamChangeToDatabaseInvoiceChange(*mdChange) - dbInvoice.Changes = append(dbInvoice.Changes, dbInvoiceChange) - - dbInvoice.Status = dbInvoiceChange.NewStatus - dbInvoice.StatusChangeReason = dbInvoiceChange.Reason - - err = c.db.UpdateInvoice(dbInvoice) - if err != nil { - return nil, err - } - - c.fireEvent(EventTypeInvoiceStatusChange, - EventDataInvoiceStatusChange{ - Invoice: dbInvoice, - AdminUser: user, - }, - ) - - // Return the reply. - sisr := v1.SetInvoiceStatusReply{ - Invoice: *convertDatabaseInvoiceToInvoice(dbInvoice), - } - return &sisr, nil -} - // HandleInvoiceDetails tries to fetch the full details of an invoice from // politeiad. func (c *cmswww) HandleInvoiceDetails( @@ -1103,6 +370,7 @@ func (c *cmswww) HandleEditInvoice( return nil, err } + log.Infof("abcde: %v", len(pdUpdateRecordReply.Record.Files)) dbInvoice, err = c.convertRecordToDatabaseInvoice( pdUpdateRecordReply.Record) if err != nil { diff --git a/cmswww/invoice_admin.go b/cmswww/invoice_admin.go new file mode 100644 index 0000000..8897886 --- /dev/null +++ b/cmswww/invoice_admin.go @@ -0,0 +1,591 @@ +package main + +import ( + "encoding/base64" + "encoding/csv" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/decred/dcrd/dcrutil" + pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/util" + + "github.com/decred/contractor-mgmt/cmswww/api/v1" + "github.com/decred/contractor-mgmt/cmswww/database" +) + +func (c *cmswww) refreshExistingInvoicePayments(dbInvoice *database.Invoice) error { + for _, dbInvoicePayment := range dbInvoice.Payments { + if dbInvoicePayment.TxID != "" { + continue + } + + dbInvoicePayment.PollExpiry = + time.Now().Add(pollExpiryDuration).Unix() + + err := c.db.UpdateInvoicePayment(dbInvoice.Token, dbInvoice.Version, + &dbInvoicePayment) + if err != nil { + return err + } + + c.addInvoicePaymentForPolling(dbInvoice.Token, &dbInvoicePayment) + } + + return nil +} + +func (c *cmswww) deriveTotalCostFromInvoice( + dbInvoice *database.Invoice, + invoicePayment *v1.InvoicePayment, +) error { + b, err := base64.StdEncoding.DecodeString(dbInvoice.Files[0].Payload) + if err != nil { + return err + } + + csvReader := csv.NewReader(strings.NewReader(string(b))) + csvReader.Comma = v1.PolicyInvoiceFieldDelimiterChar + csvReader.Comment = v1.PolicyInvoiceCommentChar + csvReader.TrimLeadingSpace = true + + records, err := csvReader.ReadAll() + if err != nil { + return err + } + + for _, record := range records { + for idx := range v1.InvoiceFields { + switch idx { + case 4: + hours, err := strconv.ParseUint(record[idx], 10, 64) + if err != nil { + return err + } + + invoicePayment.TotalHours += hours + case 5: + totalCost, err := strconv.ParseUint(record[idx], 10, 64) + if err != nil { + return err + } + + invoicePayment.TotalCostUSD += totalCost + } + } + } + + return nil +} + +func (c *cmswww) createInvoicePayment( + dbInvoice *database.Invoice, + usdDCRRate float64, + costUSD uint64, +) (*v1.InvoicePayment, error) { + invoicePayment := v1.InvoicePayment{ + UserID: strconv.FormatUint(dbInvoice.UserID, 10), + Username: dbInvoice.Username, + Token: dbInvoice.Token, + } + + var recreatingTotalCostPayment bool + dbInvoicePayment := &database.InvoicePayment{} + if costUSD == 0 { + err := c.deriveTotalCostFromInvoice(dbInvoice, &invoicePayment) + if err != nil { + return nil, err + } + + // If there's already payments on this invoice, determine + // if one of them is for the total cost. + for i, payment := range dbInvoice.Payments { + if payment.IsTotalCost { + recreatingTotalCostPayment = true + dbInvoicePayment = &dbInvoice.Payments[i] + break + } + } + + dbInvoicePayment.IsTotalCost = true + } else { + invoicePayment.TotalCostUSD = costUSD + } + + invoicePayment.TotalCostDCR = float64(invoicePayment.TotalCostUSD) / usdDCRRate + + // Generate the user's address. + user, err := c.db.GetUserById(dbInvoice.UserID) + if err != nil { + return nil, err + } + + // Create or update the invoice payment in the DB. + address, txNotBefore, err := c.derivePaymentInfo(user) + if err != nil { + return nil, err + } + + amount, err := dcrutil.NewAmount(invoicePayment.TotalCostDCR) + if err != nil { + return nil, err + } + + oldAddress := dbInvoicePayment.Address + + dbInvoicePayment.Address = address + dbInvoicePayment.TxNotBefore = txNotBefore + dbInvoicePayment.Amount = uint64(amount) + dbInvoicePayment.PollExpiry = time.Now().Add(pollExpiryDuration).Unix() + if !recreatingTotalCostPayment { + dbInvoice.Payments = append(dbInvoice.Payments, *dbInvoicePayment) + } + + err = c.updateMDPayments(dbInvoice, false, 0) + if err != nil { + return nil, err + } + + if recreatingTotalCostPayment { + err = c.db.UpdateInvoicePayment(dbInvoice.Token, dbInvoice.Version, + dbInvoicePayment) + } else { + err = c.db.UpdateInvoice(dbInvoice) + } + if err != nil { + return nil, err + } + + if recreatingTotalCostPayment { + c.removeInvoicePaymentsFromPolling([]string{oldAddress}) + } + c.addInvoicePaymentForPolling(dbInvoice.Token, dbInvoicePayment) + + invoicePayment.PaymentAddress = address + return &invoicePayment, nil +} + +func (c *cmswww) createInvoiceReview(invoice *database.Invoice) (*v1.InvoiceReview, error) { + invoiceReview := v1.InvoiceReview{ + UserID: strconv.FormatUint(invoice.UserID, 10), + Username: invoice.Username, + Token: invoice.Token, + LineItems: make([]v1.InvoiceReviewLineItem, 0), + } + + b, err := base64.StdEncoding.DecodeString(invoice.Files[0].Payload) + if err != nil { + return nil, err + } + + csvReader := csv.NewReader(strings.NewReader(string(b))) + csvReader.Comma = v1.PolicyInvoiceFieldDelimiterChar + csvReader.Comment = v1.PolicyInvoiceCommentChar + csvReader.TrimLeadingSpace = true + + records, err := csvReader.ReadAll() + if err != nil { + return nil, err + } + + for _, record := range records { + lineItem := v1.InvoiceReviewLineItem{} + for idx := range v1.InvoiceFields { + var err error + switch idx { + case 0: + lineItem.Type = record[idx] + case 1: + lineItem.Subtype = record[idx] + case 2: + lineItem.Description = record[idx] + case 3: + lineItem.Proposal = record[idx] + case 4: + lineItem.Hours, err = strconv.ParseUint(record[idx], 10, 64) + if err != nil { + return nil, err + } + + invoiceReview.TotalHours += lineItem.Hours + case 5: + lineItem.TotalCost, err = strconv.ParseUint(record[idx], 10, 64) + if err != nil { + return nil, err + } + + invoiceReview.TotalCostUSD += lineItem.TotalCost + } + } + + invoiceReview.LineItems = append(invoiceReview.LineItems, lineItem) + } + + return &invoiceReview, nil +} + +func (c *cmswww) addMDChange( + invoiceToken string, + ts int64, + status v1.InvoiceStatusT, + adminPublicKey string, + reason *string, +) (*BackendInvoiceMDChange, error) { + // Create the change record. + mdChange := BackendInvoiceMDChange{ + Version: VersionBackendInvoiceMDChange, + Timestamp: time.Now().Unix(), + NewStatus: status, + Reason: reason, + } + + blob, err := json.Marshal(mdChange) + if err != nil { + return nil, err + } + + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + + pdCommand := pd.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: invoiceToken, + MDAppend: []pd.MetadataStream{ + { + ID: mdStreamChanges, + Payload: string(blob), + }, + }, + } + + responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, + pdCommand) + if err != nil { + return nil, err + } + + var pdReply pd.UpdateVettedMetadataReply + err = json.Unmarshal(responseBody, &pdReply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "UpdateVettedMetadataReply: %v", err) + } + + // Verify the challenge. + err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) + if err != nil { + return nil, err + } + + return &mdChange, nil +} + +// fetchInvoiceFilesIfNecessary will fetch the invoice files from Politeia +// if they're not set on the invoice. If invoice files needs to be fetched, +// this function will also save them to the database. +func (c *cmswww) fetchInvoiceFilesIfNecessary(dbInvoice *database.Invoice) error { + if len(dbInvoice.Files) > 0 { + return nil + } + + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + + responseBody, err := c.rpc(http.MethodPost, pd.GetVettedRoute, + pd.GetVetted{ + Token: dbInvoice.Token, + Challenge: hex.EncodeToString(challenge), + }) + if err != nil { + return err + } + + var pdReply pd.GetVettedReply + err = json.Unmarshal(responseBody, &pdReply) + if err != nil { + return fmt.Errorf("Could not unmarshal "+ + "GetVettedReply: %v", err) + } + + // Verify the challenge. + err = util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) + if err != nil { + return err + } + + dbInvoice.Files = convertRecordFilesToDatabaseInvoiceFiles( + pdReply.Record.Files) + + return c.db.CreateInvoiceFiles(dbInvoice.Token, dbInvoice.Version, + dbInvoice.Files) +} + +// HandleSetInvoiceStatus changes the status of an existing invoice +// from unreviewed to either published or rejected. +func (c *cmswww) HandleSetInvoiceStatus( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + sis := req.(*v1.SetInvoiceStatus) + + err := checkPublicKeyAndSignature(user, sis.PublicKey, sis.Signature, + sis.Token, strconv.FormatUint(uint64(sis.Status), 10)) + if err != nil { + return nil, err + } + + dbInvoice, err := c.db.GetInvoiceByToken(sis.Token) + if err != nil { + if err == database.ErrInvoiceNotFound { + return nil, v1.UserError{ + ErrorCode: v1.ErrorStatusInvoiceNotFound, + } + } + + return nil, err + } + + err = validateStatusTransition(dbInvoice, sis.Status, sis.Reason) + if err != nil { + return nil, err + } + + log.Infof("Set invoice status, invoice has files: %v", len(dbInvoice.Files)) + + adminPublicKey, ok := database.ActiveIdentityString(user.Identities) + if !ok { + return nil, fmt.Errorf("invalid admin identity: %v", + user.ID) + } + + mdChange, err := c.addMDChange(sis.Token, time.Now().Unix(), sis.Status, + adminPublicKey, sis.Reason) + if err != nil { + return nil, err + } + + // Update the database with the metadata changes. + dbInvoiceChange := convertStreamChangeToDatabaseInvoiceChange(*mdChange) + dbInvoice.Changes = append(dbInvoice.Changes, dbInvoiceChange) + + dbInvoice.Status = dbInvoiceChange.NewStatus + dbInvoice.StatusChangeReason = dbInvoiceChange.Reason + + err = c.db.UpdateInvoice(dbInvoice) + if err != nil { + return nil, err + } + + c.fireEvent(EventTypeInvoiceStatusChange, + EventDataInvoiceStatusChange{ + Invoice: dbInvoice, + AdminUser: user, + }, + ) + + // Return the reply. + sisr := v1.SetInvoiceStatusReply{ + Invoice: *convertDatabaseInvoiceToInvoice(dbInvoice), + } + return &sisr, nil +} + +// HandleInvoices returns an array of all invoices. +func (c *cmswww) HandleInvoices( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + i := req.(*v1.Invoices) + + statusMap := make(map[v1.InvoiceStatusT]bool) + if i.Status != v1.InvoiceStatusInvalid { + statusMap[i.Status] = true + } + + invoices, numMatches, err := c.getInvoices(database.InvoicesRequest{ + Month: i.Month, + Year: i.Year, + StatusMap: statusMap, + Page: int(i.Page), + }) + if err != nil { + return nil, err + } + + return &v1.InvoicesReply{ + Invoices: invoices, + TotalMatches: uint64(numMatches), + }, nil +} + +// HandleReviewInvoices returns a list of all unreviewed invoices. +func (c *cmswww) HandleReviewInvoices( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + ri := req.(*v1.ReviewInvoices) + + invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ + Month: ri.Month, + Year: ri.Year, + StatusMap: map[v1.InvoiceStatusT]bool{ + v1.InvoiceStatusNotReviewed: true, + v1.InvoiceStatusUnreviewedChanges: true, + }, + IncludeFiles: true, + Page: -1, + }) + if err != nil { + return nil, err + } + + invoiceReviews := make([]v1.InvoiceReview, 0) + for _, invoice := range invoices { + err := c.fetchInvoiceFilesIfNecessary(&invoice) + if err != nil { + return nil, err + } + + invoiceReview, err := c.createInvoiceReview(&invoice) + if err != nil { + return nil, err + } + + invoiceReviews = append(invoiceReviews, *invoiceReview) + } + + return &v1.ReviewInvoicesReply{ + Invoices: invoiceReviews, + }, nil +} + +// HandlePayInvoices creates new invoice payments and returns their data. +func (c *cmswww) HandlePayInvoices( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + pi := req.(*v1.PayInvoices) + + invoices, _, err := c.db.GetInvoices(database.InvoicesRequest{ + Month: pi.Month, + Year: pi.Year, + StatusMap: map[v1.InvoiceStatusT]bool{ + v1.InvoiceStatusApproved: true, + }, + Page: -1, + }) + if err != nil { + return nil, err + } + + invoicePayments := make([]v1.InvoicePayment, 0) + for _, inv := range invoices { + invoice, err := c.db.GetInvoiceByToken(inv.Token) + if err != nil { + return nil, err + } + + err = c.fetchInvoiceFilesIfNecessary(invoice) + if err != nil { + return nil, err + } + + err = c.refreshExistingInvoicePayments(invoice) + if err != nil { + return nil, err + } + + invoicePayment, err := c.createInvoicePayment(invoice, pi.USDDCRRate, 0) + if err != nil { + return nil, err + } + + invoicePayments = append(invoicePayments, *invoicePayment) + } + + return &v1.PayInvoicesReply{ + Invoices: invoicePayments, + }, nil +} + +// HandlePayInvoice creates a new invoice payment and returns it. +func (c *cmswww) HandlePayInvoice( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + pi := req.(*v1.PayInvoice) + + invoice, err := c.db.GetInvoiceByToken(pi.Token) + if err != nil { + return nil, err + } + + err = c.fetchInvoiceFilesIfNecessary(invoice) + if err != nil { + return nil, err + } + + err = c.refreshExistingInvoicePayments(invoice) + if err != nil { + return nil, err + } + + invoicePayment, err := c.createInvoicePayment(invoice, pi.USDDCRRate, + pi.CostUSD) + if err != nil { + return nil, err + } + + return &v1.PayInvoiceReply{ + Invoice: *invoicePayment, + }, nil +} + +// HandleUpdateInvoicePayment updates a payment for an invoice. +func (c *cmswww) HandleUpdateInvoicePayment( + req interface{}, + user *database.User, + w http.ResponseWriter, + r *http.Request, +) (interface{}, error) { + aip := req.(*v1.UpdateInvoicePayment) + + dbInvoice, err := c.db.GetInvoiceByToken(aip.Token) + if err != nil { + if err == database.ErrInvoiceNotFound { + return nil, v1.UserError{ + ErrorCode: v1.ErrorStatusInvoiceNotFound, + } + } + + return nil, err + } + + log.Infof("Num invoice payments: %v", len(dbInvoice.Payments)) + err = c.updateInvoicePayment(dbInvoice, aip.Address, uint64(aip.Amount), + aip.TxID) + if err != nil { + return nil, err + } + + return &v1.UpdateInvoicePaymentReply{}, nil +} diff --git a/cmswww/invoice_util.go b/cmswww/invoice_util.go new file mode 100644 index 0000000..c7dfed6 --- /dev/null +++ b/cmswww/invoice_util.go @@ -0,0 +1,183 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "time" + + pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/util" + + "github.com/decred/contractor-mgmt/cmswww/api/v1" + "github.com/decred/contractor-mgmt/cmswww/database" +) + +var ( + validStatusTransitions = map[v1.InvoiceStatusT][]v1.InvoiceStatusT{ + v1.InvoiceStatusNotReviewed: { + v1.InvoiceStatusApproved, + v1.InvoiceStatusRejected, + }, + v1.InvoiceStatusRejected: { + v1.InvoiceStatusApproved, + v1.InvoiceStatusUnreviewedChanges, + }, + v1.InvoiceStatusUnreviewedChanges: { + v1.InvoiceStatusApproved, + v1.InvoiceStatusRejected, + }, + v1.InvoiceStatusApproved: { + v1.InvoiceStatusPaid, + }, + } +) + +func statusInSlice(arr []v1.InvoiceStatusT, status v1.InvoiceStatusT) bool { + for _, s := range arr { + if status == s { + return true + } + } + + return false +} + +func validateStatusTransition( + dbInvoice *database.Invoice, + newStatus v1.InvoiceStatusT, + reason *string, +) error { + validStatuses, ok := validStatusTransitions[dbInvoice.Status] + if !ok { + log.Errorf("status not supported: %v", dbInvoice.Status) + return v1.UserError{ + ErrorCode: v1.ErrorStatusInvalidInvoiceStatusTransition, + } + } + + if !statusInSlice(validStatuses, newStatus) { + return v1.UserError{ + ErrorCode: v1.ErrorStatusInvalidInvoiceStatusTransition, + } + } + + if newStatus == v1.InvoiceStatusRejected && reason == nil { + return v1.UserError{ + ErrorCode: v1.ErrorStatusReasonNotProvided, + } + } + + return nil +} + +func (c *cmswww) updateMDPayments( + dbInvoice *database.Invoice, + updatingInvoicePayment bool, + ts int64, +) error { + // Create the payments metadata record. + mdPayments, err := convertDatabaseInvoicePaymentsToStreamPayments(dbInvoice) + if err != nil { + return err + } + + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return fmt.Errorf("could not create challenge: %v", err) + } + + pdCommand := pd.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: dbInvoice.Token, + MDOverwrite: []pd.MetadataStream{ + { + ID: mdStreamPayments, + Payload: mdPayments, + }, + }, + } + + // Create the change metadata record if an existing invoice payment + // is being updated. + if updatingInvoicePayment && dbInvoice.Status != v1.InvoiceStatusPaid { + mdChange, err := json.Marshal(BackendInvoiceMDChange{ + Version: VersionBackendInvoiceMDChange, + Timestamp: ts, + NewStatus: v1.InvoiceStatusPaid, + }) + if err != nil { + return fmt.Errorf("cannot marshal backend change: %v", err) + } + + pdCommand.MDAppend = []pd.MetadataStream{ + { + ID: mdStreamChanges, + Payload: string(mdChange), + }, + } + } + + responseBody, err := c.rpc(http.MethodPost, pd.UpdateVettedMetadataRoute, + pdCommand) + if err != nil { + return err + } + + var pdReply pd.UpdateVettedMetadataReply + err = json.Unmarshal(responseBody, &pdReply) + if err != nil { + return fmt.Errorf("Could not unmarshal UpdateVettedMetadataReply: %v", + err) + } + + // Verify the challenge. + return util.VerifyChallenge(c.cfg.Identity, challenge, pdReply.Response) +} + +func (c *cmswww) updateInvoicePayment( + dbInvoice *database.Invoice, + address string, + amount uint64, + txID string, +) error { + var dbInvoicePayment *database.InvoicePayment + for idx, payment := range dbInvoice.Payments { + if payment.Amount == amount && payment.Address == address { + dbInvoice.Payments[idx].TxID = txID + dbInvoicePayment = &payment + break + } + } + + if dbInvoicePayment == nil { + return v1.UserError{ + ErrorCode: v1.ErrorStatusInvoicePaymentNotFound, + } + } + + // Update the Politeia record. + ts := time.Now().Unix() + err := c.updateMDPayments(dbInvoice, true, ts) + if err != nil { + return err + } + + // Update the invoice in the database. + if dbInvoice.Status != v1.InvoiceStatusPaid { + // Update the status in the database if necessary. + dbInvoice.Status = v1.InvoiceStatusPaid + dbInvoice.Changes = append(dbInvoice.Changes, database.InvoiceChange{ + Timestamp: ts, + NewStatus: v1.InvoiceStatusPaid, + }) + } + err = c.db.UpdateInvoice(dbInvoice) + if err != nil { + return fmt.Errorf("cannot update invoice with token %v: %v", + dbInvoice.Token, err) + } + + return nil +} diff --git a/cmswww/payment.go b/cmswww/payment.go index 28a0250..a0e7544 100644 --- a/cmswww/payment.go +++ b/cmswww/payment.go @@ -99,7 +99,8 @@ func (c *cmswww) addInvoicePaymentsForPolling() error { invoicePayment.PollExpiry = time.Now().Add(pollExpiryDuration).Unix() - err = c.db.UpdateInvoicePayment(&invoicePayment) + err = c.db.UpdateInvoicePayment(invoice.Token, invoice.Version, + &invoicePayment) if err != nil { return err } From 1c1ab7a9f37152f32f1b38bc81d01aa16c20db3c Mon Sep 17 00:00:00 2001 From: Sean Durkin Date: Mon, 25 Feb 2019 22:51:41 -0600 Subject: [PATCH 5/5] Fix regression. --- cmswww/cmd/cmswwwcli/commands/changepassword.go | 4 ++-- cmswww/cmd/cmswwwcli/commands/resetpassword.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmswww/cmd/cmswwwcli/commands/changepassword.go b/cmswww/cmd/cmswwwcli/commands/changepassword.go index e8cae7b..a436bd7 100644 --- a/cmswww/cmd/cmswwwcli/commands/changepassword.go +++ b/cmswww/cmd/cmswwwcli/commands/changepassword.go @@ -33,8 +33,8 @@ func (cmd *ChangePasswordCmd) Execute(args []string) error { // Send the change password request. cp := v1.ChangePassword{ - CurrentPassword: cmd.Args.CurrentPassword, - NewPassword: cmd.Args.NewPassword, + CurrentPassword: DigestSHA3(cmd.Args.CurrentPassword), + NewPassword: DigestSHA3(cmd.Args.NewPassword), } var cpr v1.ChangePasswordReply diff --git a/cmswww/cmd/cmswwwcli/commands/resetpassword.go b/cmswww/cmd/cmswwwcli/commands/resetpassword.go index 9958f5f..daaf04e 100644 --- a/cmswww/cmd/cmswwwcli/commands/resetpassword.go +++ b/cmswww/cmd/cmswwwcli/commands/resetpassword.go @@ -51,7 +51,7 @@ func (cmd *ResetPasswordCmd) Execute(args []string) error { if cmd.Token != "" { rp.VerificationToken = cmd.Token - rp.NewPassword = cmd.NewPassword + rp.NewPassword = DigestSHA3(cmd.NewPassword) } var rpr v1.ResetPasswordReply @@ -65,7 +65,7 @@ func (cmd *ResetPasswordCmd) Execute(args []string) error { // Automatic 2nd reset password call rp = &v1.ResetPassword{ Email: cmd.Args.Email, - NewPassword: cmd.NewPassword, + NewPassword: DigestSHA3(cmd.NewPassword), VerificationToken: rpr.VerificationToken, }