Skip to content

Commit

Permalink
Moved document, revision, attachment stuff into new docmodel package
Browse files Browse the repository at this point in the history
  • Loading branch information
@snej
snej committed Feb 23, 2023
1 parent ff2c7e9 commit 5df9a94
Show file tree
Hide file tree
Showing 47 changed files with 1,311 additions and 1,142 deletions.
170 changes: 17 additions & 153 deletions db/attachment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,16 @@ package db

import (
"context"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"net/http"

"github.com/couchbase/sync_gateway/base"
"github.com/couchbase/sync_gateway/docmodel"
)

const (
// AttVersion1 attachments are persisted to the bucket based on attachment body digest.
AttVersion1 int = 1

// AttVersion2 attachments are persisted to the bucket based on docID and body digest.
AttVersion2 int = 2
)
const maxAttachmentSizeBytes = 20 * 1024 * 1024

var (
// ErrAttachmentVersion is thrown in case of any error in parsing version from the attachment meta.
Expand All @@ -37,29 +29,6 @@ var (
ErrAttachmentMeta = base.HTTPErrorf(http.StatusBadRequest, "Invalid _attachments")
)

// AttachmentData holds the attachment key and value bytes.
type AttachmentData map[string][]byte

// A map of keys -> DocAttachments.
type AttachmentMap map[string]*DocAttachment

// A struct which models an attachment. Currently only used by test code, however
// new code or refactoring in the main codebase should try to use where appropriate.
type DocAttachment struct {
ContentType string `json:"content_type,omitempty"`
Digest string `json:"digest,omitempty"`
Length int `json:"length,omitempty"`
Revpos int `json:"revpos,omitempty"`
Stub bool `json:"stub,omitempty"`
Version int `json:"ver,omitempty"`
Data []byte `json:"-"` // tell json marshal/unmarshal to ignore this field
}

// ErrAttachmentTooLarge is returned when an attempt to attach an oversize attachment is made.
var ErrAttachmentTooLarge = errors.New("attachment too large")

const maxAttachmentSizeBytes = 20 * 1024 * 1024

// Given Attachments Meta to be stored in the database, storeAttachments goes through the map, finds attachments with
// inline bodies, copies the bodies into the Couchbase db, and replaces the bodies with the 'digest' attributes which
// are the keys to retrieving them.
Expand All @@ -79,19 +48,19 @@ func (db *DatabaseCollectionWithUser) storeAttachments(ctx context.Context, doc
data := meta["data"]
if data != nil {
// Attachment contains data, so store it in the db:
attachment, err := DecodeAttachment(data)
attachment, err := docmodel.DecodeAttachment(data)
if err != nil {
return nil, err
}
digest := Sha1DigestKey(attachment)
key := MakeAttachmentKey(AttVersion2, doc.ID, digest)
key := MakeAttachmentKey(docmodel.AttVersion2, doc.ID, digest)
newAttachmentData[key] = attachment

newMeta := map[string]interface{}{
"stub": true,
"digest": digest,
"revpos": generation,
"ver": AttVersion2,
"ver": docmodel.AttVersion2,
}
if contentType, ok := meta["content_type"].(string); ok {
newMeta["content_type"] = contentType
Expand Down Expand Up @@ -146,7 +115,7 @@ func retrieveV2AttachmentKeys(docID string, docAttachments AttachmentsMeta) (att
if !ok {
return nil, ErrAttachmentMeta
}
version, _ := GetAttachmentVersion(meta)
version, _ := docmodel.GetAttachmentVersion(meta)
if version != AttVersion2 {
continue
}
Expand All @@ -167,10 +136,10 @@ func (db *DatabaseCollectionWithUser) retrieveAncestorAttachments(ctx context.Co
}

// No non-pruned ancestor is available
if commonAncestor := doc.History.findAncestorFromSet(doc.CurrentRev, docHistory); commonAncestor != "" {
if commonAncestor := doc.History.FindAncestorFromSet(doc.CurrentRev, docHistory); commonAncestor != "" {
parentAttachments := make(map[string]interface{})
commonAncestorGen := int64(genOfRevID(commonAncestor))
for name, activeAttachment := range GetBodyAttachments(doc.Body()) {
commonAncestorGen := int64(docmodel.GenOfRevID(commonAncestor))
for name, activeAttachment := range docmodel.GetBodyAttachments(doc.Body()) {
if attachmentMeta, ok := activeAttachment.(map[string]interface{}); ok {
activeRevpos, ok := base.ToInt64(attachmentMeta["revpos"])
if ok && activeRevpos <= commonAncestorGen {
Expand Down Expand Up @@ -204,7 +173,7 @@ func (c *DatabaseCollection) loadAttachmentsData(attachments AttachmentsMeta, mi
if !ok {
return nil, base.RedactErrorf("Unable to load attachment for doc: %v with name: %v and revpos: %v due to unexpected digest field: %v", base.UD(docid), base.UD(attachmentName), revpos, digest)
}
version, ok := GetAttachmentVersion(meta)
version, ok := docmodel.GetAttachmentVersion(meta)
if !ok {
return nil, base.RedactErrorf("Unable to load attachment for doc: %v with name: %v, revpos: %v and digest: %v due to unexpected version value: %v", base.UD(docid), base.UD(attachmentName), revpos, digest, version)
}
Expand All @@ -221,14 +190,6 @@ func (c *DatabaseCollection) loadAttachmentsData(attachments AttachmentsMeta, mi
return newAttachments, nil
}

// DeleteAttachmentVersion removes attachment versions from the AttachmentsMeta map specified.
func DeleteAttachmentVersion(attachments AttachmentsMeta) {
for _, value := range attachments {
meta := value.(map[string]interface{})
delete(meta, "ver")
}
}

// GetAttachment retrieves an attachment given its key.
func (c *DatabaseCollection) GetAttachment(key string) ([]byte, error) {
v, _, err := c.dataStore.GetRaw(key)
Expand All @@ -248,7 +209,7 @@ func (db *DatabaseCollectionWithUser) setAttachments(ctx context.Context, attach
for key, data := range attachments {
attachmentSize := int64(len(data))
if attachmentSize > int64(maxAttachmentSizeBytes) {
return ErrAttachmentTooLarge
return docmodel.ErrAttachmentTooLarge
}
_, err := db.dataStore.AddRaw(key, 0, data)
if err == nil {
Expand All @@ -270,7 +231,7 @@ type AttachmentCallback func(name string, digest string, knownData []byte, meta
// to its digest. If the attachment isn't known, the callback can return data for it, which will
// be added to the metadata as a "data" property.
func (c *DatabaseCollection) ForEachStubAttachment(body Body, minRevpos int, docID string, existingDigests map[string]string, callback AttachmentCallback) error {
atts := GetBodyAttachments(body)
atts := docmodel.GetBodyAttachments(body)
if atts == nil && body[BodyAttachments] != nil {
return base.HTTPErrorf(http.StatusBadRequest, "Invalid _attachments")
}
Expand Down Expand Up @@ -319,103 +280,14 @@ func (c *DatabaseCollection) ForEachStubAttachment(body Body, minRevpos int, doc
return nil
}

func GetAttachmentVersion(meta map[string]interface{}) (int, bool) {
ver, ok := meta["ver"]
if !ok {
return AttVersion1, true
}
val, ok := base.ToInt64(ver)
return int(val), ok
}

// GenerateProofOfAttachment returns a nonce and proof for an attachment body.
func GenerateProofOfAttachment(attachmentData []byte) (nonce []byte, proof string, err error) {
nonce = make([]byte, 20)
if _, err := rand.Read(nonce); err != nil {
return nil, "", base.HTTPErrorf(http.StatusInternalServerError, fmt.Sprintf("Failed to generate random data: %s", err))
}
proof = ProveAttachment(attachmentData, nonce)
base.TracefCtx(context.Background(), base.KeyCRUD, "Generated nonce %v and proof %q for attachment: %v", nonce, proof, attachmentData)
return nonce, proof, nil
}

// ProveAttachment returns the proof for an attachment body and nonce pair.
func ProveAttachment(attachmentData, nonce []byte) (proof string) {
d := sha1.New()
d.Write([]byte{byte(len(nonce))})
d.Write(nonce)
d.Write(attachmentData)
proof = "sha1-" + base64.StdEncoding.EncodeToString(d.Sum(nil))
base.TracefCtx(context.Background(), base.KeyCRUD, "Generated proof %q using nonce %v for attachment: %v", proof, nonce, attachmentData)
return proof
}

// ////// HELPERS:
// Returns _attachments property from body, when found. Checks for either map[string]interface{} (unmarshalled with body),
// or AttachmentsMeta (written by body by SG)
func GetBodyAttachments(body Body) AttachmentsMeta {
switch atts := body[BodyAttachments].(type) {
case AttachmentsMeta:
return atts
case map[string]interface{}:
return AttachmentsMeta(atts)
default:
return nil
}
}

// AttachmentDigests returns a list of attachment digests contained in the given AttachmentsMeta
func AttachmentDigests(attachments AttachmentsMeta) []string {
var digests = make([]string, 0, len(attachments))
for _, att := range attachments {
if attMap, ok := att.(map[string]interface{}); ok {
if digest, ok := attMap["digest"]; ok {
if digestString, ok := digest.(string); ok {
digests = append(digests, digestString)
}
}
}
}
return digests
}

// AttachmentStorageMeta holds the metadata for building
// the key for attachment storage and retrieval.
type AttachmentStorageMeta struct {
digest string
version int
}

// ToAttachmentStorageMeta returns a slice of AttachmentStorageMeta, which is contains the
// necessary metadata properties to build the key for attachment storage and retrieval.
func ToAttachmentStorageMeta(attachments AttachmentsMeta) []AttachmentStorageMeta {
meta := make([]AttachmentStorageMeta, 0, len(attachments))
for _, att := range attachments {
if attMap, ok := att.(map[string]interface{}); ok {
if digest, ok := attMap["digest"]; ok {
if digestString, ok := digest.(string); ok {
version, _ := GetAttachmentVersion(attMap)
m := AttachmentStorageMeta{
digest: digestString,
version: version,
}
meta = append(meta, m)
}
}
}
}
return meta
}

func DecodeAttachment(att interface{}) ([]byte, error) {
switch att := att.(type) {
case []byte:
return att, nil
case string:
return base64.StdEncoding.DecodeString(att)
default:
return nil, base.HTTPErrorf(400, "invalid attachment data (type %T)", att)
// MakeAttachmentKey returns the unique for attachment storage and retrieval.
func MakeAttachmentKey(version int, docID, digest string) string {
if version == AttVersion2 {
return base.Att2Prefix + sha256Digest([]byte(docID)) + ":" + digest
}
return base.AttPrefix + digest
}

func Sha1DigestKey(data []byte) string {
Expand All @@ -424,14 +296,6 @@ func Sha1DigestKey(data []byte) string {
return "sha1-" + base64.StdEncoding.EncodeToString(digester.Sum(nil))
}

// MakeAttachmentKey returns the unique for attachment storage and retrieval.
func MakeAttachmentKey(version int, docID, digest string) string {
if version == AttVersion2 {
return base.Att2Prefix + sha256Digest([]byte(docID)) + ":" + digest
}
return base.AttPrefix + digest
}

// sha256Digest returns sha256 digest of the input bytes encoded
// by using the standard base64 encoding, as defined in RFC 4648.
func sha256Digest(key []byte) string {
Expand Down
9 changes: 5 additions & 4 deletions db/attachment_compaction.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
sgbucket "github.com/couchbase/sg-bucket"
"github.com/couchbase/sync_gateway/base"
"github.com/couchbase/sync_gateway/channels"
"github.com/couchbase/sync_gateway/docmodel"
)

const (
Expand Down Expand Up @@ -211,7 +212,7 @@ func getAttachmentSyncData(dataType uint8, data []byte) (*AttachmentCompactionDa
var documentBody []byte

if dataType&base.MemcachedDataTypeXattr != 0 {
body, xattr, _, err := parseXattrStreamData(base.SyncXattrName, "", data)
body, xattr, _, err := docmodel.ParseXattrStreamData(base.SyncXattrName, "", data)
if err != nil {
if errors.Is(err, base.ErrXattrNotFound) {
return nil, nil
Expand Down Expand Up @@ -275,7 +276,7 @@ func handleAttachments(attachmentKeyMap map[string]string, docKey string, attach
for attName, attachmentMeta := range attachmentsMap {
attMetaMap := attachmentMeta

attVer, ok := GetAttachmentVersion(attMetaMap)
attVer, ok := docmodel.GetAttachmentVersion(attMetaMap)
if !ok {
continue
}
Expand Down Expand Up @@ -312,7 +313,7 @@ func attachmentCompactSweepPhase(ctx context.Context, dataStore base.DataStore,

// If the data contains an xattr then the attachment likely has a compaction ID, need to check this value
if event.DataType&base.MemcachedDataTypeXattr != 0 {
_, xattr, _, err := parseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value)
_, xattr, _, err := docmodel.ParseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value)
if err != nil && !errors.Is(err, base.ErrXattrNotFound) {
base.WarnfCtx(ctx, "[%s] Unexpected error occurred attempting to parse attachment xattr: %v", compactionLoggingID, err)
return true
Expand Down Expand Up @@ -424,7 +425,7 @@ func attachmentCompactCleanupPhase(ctx context.Context, dataStore base.DataStore
return true
}

_, xattr, _, err := parseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value)
_, xattr, _, err := docmodel.ParseXattrStreamData(base.AttachmentCompactionXattrName, "", event.Value)
if err != nil && !errors.Is(err, base.ErrXattrNotFound) {
base.WarnfCtx(ctx, "[%s] Unexpected error occurred attempting to parse attachment xattr: %v", compactionLoggingID, err)
return true
Expand Down
Loading

0 comments on commit 5df9a94

Please sign in to comment.