diff --git a/go.mod b/go.mod index fd467e84e..73f2696ff 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/RoaringBitmap/roaring v1.9.3 github.com/google/uuid v1.6.0 github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15 - github.com/longhorn/go-common-libs v0.0.0-20240514074907-351459694cbf + github.com/longhorn/go-common-libs v0.0.0-20240605094403-68743dacbcb1 github.com/longhorn/go-spdk-helper v0.0.0-20240514082311-4069f4804017 - github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240514224711-e39b7f0313b2 + github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240612062520-b044fa75d9e5 github.com/longhorn/longhorn-spdk-engine v0.0.0-20240516014845-759dfa872eae - github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 + github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/urfave/cli v1.22.15 diff --git a/go.sum b/go.sum index 75dede89e..e1ab4aa26 100644 --- a/go.sum +++ b/go.sum @@ -67,22 +67,22 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15 h1:D838/RWPKmP9ozjOM2Al8s11NbTGLC2m+ZKT3JAogfY= github.com/longhorn/backupstore v0.0.0-20240509144945-3bce6e69af15/go.mod h1:n210xpMUVrSn/W4Za/9BZhyXLCTVfAOq5lNdLrRSyz8= -github.com/longhorn/go-common-libs v0.0.0-20240514074907-351459694cbf h1:3mjybe5dP3uJF2daMVv/U/IGNcp1dsrzHl4b6w8icPU= -github.com/longhorn/go-common-libs v0.0.0-20240514074907-351459694cbf/go.mod h1:gFXUEciTv/03ncyA8CNrfVkbikvSWNqCYwwsTC3lFGg= +github.com/longhorn/go-common-libs v0.0.0-20240605094403-68743dacbcb1 h1:Q0/LpqfbGG++p1S+3LaYmEWD5+QTP/hZou2f861zUMM= +github.com/longhorn/go-common-libs v0.0.0-20240605094403-68743dacbcb1/go.mod h1:C1B3gmrgMK2bgQhU7wntbFy6PiJDucNFRcoQKjrTu8k= github.com/longhorn/go-iscsi-helper v0.0.0-20240427164656-e9439c0018ce h1:PxKniE9F6IZ2DMKfmxDsbqeAxQI1TZhnw7/HOBMs1Is= github.com/longhorn/go-iscsi-helper v0.0.0-20240427164656-e9439c0018ce/go.mod h1:d9t3gtE+UPjescbCFluXd4xBc8OQT/JrC2cdkk2IXWQ= github.com/longhorn/go-spdk-helper v0.0.0-20240514082311-4069f4804017 h1:Cl8mEO7uP2T9ZDbqV3BK2xhrBVyjelY0WV9LDGdqb/Y= github.com/longhorn/go-spdk-helper v0.0.0-20240514082311-4069f4804017/go.mod h1:SgKStbsHMJqVjf/uDpS7KYQakOp2jHBRSbRnMk6iNqo= -github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240514224711-e39b7f0313b2 h1:mG7wUWZrJ5arEZWaghm73ywH09LUZo77kGUaHN7nzyg= -github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240514224711-e39b7f0313b2/go.mod h1:G+iGNlJm1RKuKXqz4Igh0b5miiRblld2C9GL9hs4ROk= +github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240612062520-b044fa75d9e5 h1:n9Yvhe8KG3kKWjWxvW5rs+CSqsrXwe3fGr6MMFJbeqI= +github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240612062520-b044fa75d9e5/go.mod h1:bD3PYhTXGh1nFOQUX01BeNyNJcbX8kWf8/1cFrgHwW8= github.com/longhorn/longhorn-spdk-engine v0.0.0-20240516014845-759dfa872eae h1:07uvZWinlp6Zwvuhacv9Lv3NzRM46UKPu0xIAXP7VEA= github.com/longhorn/longhorn-spdk-engine v0.0.0-20240516014845-759dfa872eae/go.mod h1:i8arwLcEawuN9rdHmo3doVhg9fQFPh0q+GJaKR2FOxk= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003 h1:Jw9uANsGcHTxp6HcC++/vN17LfeuDmozHI2j6DoZf5E= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8 h1:lwtmZEomiv8uchwo9JIyoo+lK8J3cLCm7/qzpn6wmzo= github.com/longhorn/sparse-tools v0.0.0-20240427164751-a7b9f1b2c8a8/go.mod h1:pvlUkVwRGojXhcTkkzksOe4i7GVk59P2PbJjHIB2Yj0= -github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 h1:yCn2uYikI3xW3i1HHpKytitJ2lc7S5obDMKIa7ZIuc8= -github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3/go.mod h1:1oEh1cnDDqNSuFh/dH/lvJ3Ssq83SOweTAAPLRY4PMI= +github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd h1:gzvHnEc4vdHmOtxwgjC/7YmChbzDsfYiY0wpI3RgB1A= +github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd/go.mod h1:1oEh1cnDDqNSuFh/dH/lvJ3Ssq83SOweTAAPLRY4PMI= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go index 88c03cc99..520d92681 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/backup.go @@ -1,6 +1,7 @@ package replica import ( + "context" "fmt" "os" "sync" @@ -93,7 +94,7 @@ func (rb *BackupStatus) OpenSnapshot(snapID, volumeID string) error { if err != nil { return errors.Wrap(err, "cannot get working directory") } - r, err := NewReadOnly(dir, id, rb.backingFile) + r, err := NewReadOnly(context.Background(), dir, id, rb.backingFile) if err != nil { return err } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go index c43fb51de..815b9bbcd 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/replica.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "github.com/pkg/errors" @@ -43,6 +44,7 @@ var ( type Replica struct { sync.RWMutex + ctx context.Context volume diffDisk dir string info Info @@ -53,11 +55,11 @@ type Replica struct { activeDiskData []*disk readOnly bool - revisionLock sync.Mutex - revisionCache int64 + revisionCache atomic.Int64 revisionFile *sparse.DirectFileIoProcessor - revisionRefreshed bool revisionCounterDisabled bool + revisionCounterReqChan chan bool + revisionCounterAckChan chan error unmapMarkDiskChainRemoved bool @@ -129,7 +131,7 @@ func OpenSnapshot(dir string, snapshotName string) (*Replica, error) { } } - r, err := NewReadOnly(dir, snapshotDiskName, backingFile) + r, err := NewReadOnly(context.Background(), dir, snapshotDiskName, backingFile) if err != nil { return nil, err } @@ -142,17 +144,17 @@ func ReadInfo(dir string) (Info, error) { return info, err } -func New(size, sectorSize int64, dir string, backingFile *backingfile.BackingFile, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, SnapshotMaxSize int64) (*Replica, error) { - return construct(false, size, sectorSize, dir, "", backingFile, disableRevCounter, unmapMarkDiskChainRemoved, snapshotMaxCount, SnapshotMaxSize) +func New(ctx context.Context, size, sectorSize int64, dir string, backingFile *backingfile.BackingFile, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, SnapshotMaxSize int64) (*Replica, error) { + return construct(ctx, false, size, sectorSize, dir, "", backingFile, disableRevCounter, unmapMarkDiskChainRemoved, snapshotMaxCount, SnapshotMaxSize) } -func NewReadOnly(dir, head string, backingFile *backingfile.BackingFile) (*Replica, error) { +func NewReadOnly(ctx context.Context, dir, head string, backingFile *backingfile.BackingFile) (*Replica, error) { // size and sectorSize don't matter because they will be read from metadata // snapshotMaxCount and SnapshotMaxSize don't matter because readonly replica can't create a new disk - return construct(true, 0, diskutil.ReplicaSectorSize, dir, head, backingFile, false, false, 250, 0) + return construct(ctx, true, 0, diskutil.ReplicaSectorSize, dir, head, backingFile, false, false, 250, 0) } -func construct(readonly bool, size, sectorSize int64, dir, head string, backingFile *backingfile.BackingFile, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, snapshotMaxSize int64) (*Replica, error) { +func construct(ctx context.Context, readonly bool, size, sectorSize int64, dir, head string, backingFile *backingfile.BackingFile, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, snapshotMaxSize int64) (*Replica, error) { if size%sectorSize != 0 { return nil, fmt.Errorf("size %d not a multiple of sector size %d", size, sectorSize) } @@ -162,12 +164,15 @@ func construct(readonly bool, size, sectorSize int64, dir, head string, backingF } r := &Replica{ + ctx: ctx, dir: dir, activeDiskData: make([]*disk, 1), diskData: make(map[string]*disk), diskChildrenMap: map[string]map[string]bool{}, readOnly: readonly, revisionCounterDisabled: disableRevCounter, + revisionCounterReqChan: make(chan bool, 1024), // Buffered channel to avoid blocking + revisionCounterAckChan: make(chan error), unmapMarkDiskChainRemoved: unmapMarkDiskChainRemoved, snapshotMaxCount: snapshotMaxCount, snapshotMaxSize: snapshotMaxSize, @@ -195,7 +200,7 @@ func construct(readonly bool, size, sectorSize int64, dir, head string, backingF } if !r.revisionCounterDisabled { - if err := r.initRevisionCounter(); err != nil { + if err := r.initRevisionCounter(ctx); err != nil { return nil, err } } @@ -289,7 +294,7 @@ func (r *Replica) SetRebuilding(rebuilding bool) error { } func (r *Replica) Reload() (*Replica, error) { - newReplica, err := New(r.info.Size, r.info.SectorSize, r.dir, r.info.BackingFile, r.revisionCounterDisabled, r.unmapMarkDiskChainRemoved, r.snapshotMaxCount, r.snapshotMaxSize) + newReplica, err := New(r.ctx, r.info.Size, r.info.SectorSize, r.dir, r.info.BackingFile, r.revisionCounterDisabled, r.unmapMarkDiskChainRemoved, r.snapshotMaxCount, r.snapshotMaxSize) if err != nil { return nil, err } @@ -1275,6 +1280,15 @@ func (r *Replica) WriteAt(buf []byte, offset int64) (int, error) { return 0, fmt.Errorf("cannot write on read-only replica") } + // Increase the revision counter optimistically in a separate goroutine since most of the time write operations will succeed. + // Once the write operation fails, the revision counter will be wrongly increased by 1. It means that the revision counter is not accurate. + // Actually, the revision counter is not accurate even without this optimistic increment since we cannot make data write operation and the revision counter increment atomic. + go func() { + if !r.revisionCounterDisabled { + r.revisionCounterReqChan <- true + } + }() + r.RLock() r.info.Dirty = true c, err := r.volume.WriteAt(buf, offset) @@ -1284,12 +1298,10 @@ func (r *Replica) WriteAt(buf []byte, offset int64) (int, error) { } if !r.revisionCounterDisabled { - if err := r.increaseRevisionCounter(); err != nil { - return c, err - } + err = <-r.revisionCounterAckChan } - return c, nil + return c, err } func (r *Replica) ReadAt(buf []byte, offset int64) (int, error) { @@ -1309,6 +1321,12 @@ func (r *Replica) UnmapAt(length uint32, offset int64) (n int, err error) { return 0, fmt.Errorf("can not unmap on read-only replica") } + go func() { + if !r.revisionCounterDisabled { + r.revisionCounterReqChan <- true + } + }() + unmappedSize, err := func() (int, error) { r.Lock() defer r.Unlock() @@ -1346,12 +1364,10 @@ func (r *Replica) UnmapAt(length uint32, offset int64) (n int, err error) { } if !r.revisionCounterDisabled { - if err := r.increaseRevisionCounter(); err != nil { - return 0, err - } + err = <-r.revisionCounterAckChan } - return unmappedSize, nil + return unmappedSize, err } func (r *Replica) ListDisks() map[string]DiskInfo { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go index e6b682e10..dc9514eec 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/revision_counter.go @@ -1,6 +1,7 @@ package replica import ( + "context" "fmt" "io" "os" @@ -8,7 +9,6 @@ import ( "strings" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/longhorn/sparse-tools/sparse" ) @@ -63,14 +63,11 @@ func (r *Replica) openRevisionFile(isCreate bool) error { return err } -func (r *Replica) initRevisionCounter() error { +func (r *Replica) initRevisionCounter(ctx context.Context) error { if r.readOnly { return nil } - r.revisionLock.Lock() - defer r.revisionLock.Unlock() - if _, err := os.Stat(r.diskPath(revisionCounterFile)); err != nil { if !os.IsNotExist(err) { return err @@ -90,11 +87,31 @@ func (r *Replica) initRevisionCounter() error { if err != nil { return err } - // Don't use r.revisionCache directly - // r.revisionCache is an internal cache, to avoid read from disk - // every time when counter needs to be updated. - // And it's protected by revisionLock - r.revisionCache = counter + // To avoid read from disk, apply atomic operations against the internal cache r.revisionCache every time counter needs to be updated + r.revisionCache.Swap(counter) + + go func() { + if r.revisionCounterDisabled { + return + } + var err error + for { + select { + case <-ctx.Done(): + return + case <-r.revisionCounterReqChan: + err = r.writeRevisionCounter(r.revisionCache.Load() + 1) + r.revisionCounterAckChan <- err + if err != nil { + close(r.revisionCounterAckChan) + return + } else { + r.revisionCache.Add(1) + } + } + } + }() + return nil } @@ -103,50 +120,17 @@ func (r *Replica) IsRevCounterDisabled() bool { } func (r *Replica) GetRevisionCounter() int64 { - r.revisionLock.Lock() - defer r.revisionLock.Unlock() - - counter, err := r.readRevisionCounter() - if err != nil { - logrus.WithError(err).Error("Failed to get revision counter") - // -1 will result in the replica to be discarded - return -1 - } - r.revisionCache = counter - return counter + return r.revisionCache.Load() } +// SetRevisionCounter can be invoked only when there is no pending IO. +// Typically, its caller, the engine process, will hold the lock before calling this function. +// And the engine lock holding means all writable replicas finished their IO. +// In other words, the engine lock holding means there is no pending IO. func (r *Replica) SetRevisionCounter(counter int64) error { - r.revisionLock.Lock() - defer r.revisionLock.Unlock() - if err := r.writeRevisionCounter(counter); err != nil { return err } - - r.revisionCache = counter - return nil -} - -func (r *Replica) increaseRevisionCounter() error { - r.revisionLock.Lock() - defer r.revisionLock.Unlock() - - if !r.revisionRefreshed { - counter, err := r.readRevisionCounter() - if err != nil { - return err - } - logrus.Infof("Reloading the revision counter before processing the first write, the current revision cache is %v, the latest revision counter in file is %v", - r.revisionCache, counter) - r.revisionCache = counter - r.revisionRefreshed = true - } - - if err := r.writeRevisionCounter(r.revisionCache + 1); err != nil { - return err - } - - r.revisionCache++ + r.revisionCache.Swap(counter) return nil } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go index a781385d6..c1ce6094d 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/replica/server.go @@ -1,6 +1,7 @@ package replica import ( + "context" "fmt" "os" "sync" @@ -13,6 +14,7 @@ import ( type Server struct { sync.RWMutex + ctx context.Context r *Replica dir string sectorSize int64 @@ -23,8 +25,9 @@ type Server struct { snapshotMaxSize int64 } -func NewServer(dir string, backing *backingfile.BackingFile, sectorSize int64, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, snapshotMaxSize int64) *Server { +func NewServer(ctx context.Context, dir string, backing *backingfile.BackingFile, sectorSize int64, disableRevCounter, unmapMarkDiskChainRemoved bool, snapshotMaxCount int, snapshotMaxSize int64) *Server { return &Server{ + ctx: ctx, dir: dir, backing: backing, sectorSize: sectorSize, @@ -54,7 +57,7 @@ func (s *Server) Create(size int64) error { sectorSize := s.getSectorSize() logrus.Infof("Creating replica %s, size %d/%d", s.dir, size, sectorSize) - r, err := New(size, sectorSize, s.dir, s.backing, s.revisionCounterDisabled, s.unmapMarkDiskChainRemoved, s.snapshotMaxCount, s.snapshotMaxSize) + r, err := New(s.ctx, size, sectorSize, s.dir, s.backing, s.revisionCounterDisabled, s.unmapMarkDiskChainRemoved, s.snapshotMaxCount, s.snapshotMaxSize) if err != nil { return err } @@ -74,7 +77,7 @@ func (s *Server) Open() error { sectorSize := s.getSectorSize() logrus.Infof("Opening replica: dir %s, size %d, sector size %d", s.dir, info.Size, sectorSize) - r, err := New(info.Size, sectorSize, s.dir, s.backing, s.revisionCounterDisabled, s.unmapMarkDiskChainRemoved, s.snapshotMaxCount, s.snapshotMaxSize) + r, err := New(s.ctx, info.Size, sectorSize, s.dir, s.backing, s.revisionCounterDisabled, s.unmapMarkDiskChainRemoved, s.snapshotMaxCount, s.snapshotMaxSize) if err != nil { return err } diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go index d93911cd6..3c5ec3ce5 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go @@ -138,7 +138,8 @@ type Replica struct { } type ReplicaSalvageInfo struct { - LastModifyTime int64 + Address string + LastModifyTime time.Time HeadFileSize int64 } diff --git a/vendor/modules.txt b/vendor/modules.txt index 0757e4223..46e7699ba 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -142,7 +142,7 @@ github.com/longhorn/backupstore/s3 github.com/longhorn/backupstore/types github.com/longhorn/backupstore/util github.com/longhorn/backupstore/vfs -# github.com/longhorn/go-common-libs v0.0.0-20240514074907-351459694cbf +# github.com/longhorn/go-common-libs v0.0.0-20240605094403-68743dacbcb1 ## explicit; go 1.22.0 github.com/longhorn/go-common-libs/exec github.com/longhorn/go-common-libs/io @@ -169,7 +169,7 @@ github.com/longhorn/go-spdk-helper/pkg/spdk/setup github.com/longhorn/go-spdk-helper/pkg/spdk/types github.com/longhorn/go-spdk-helper/pkg/types github.com/longhorn/go-spdk-helper/pkg/util -# github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240514224711-e39b7f0313b2 +# github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240612062520-b044fa75d9e5 ## explicit; go 1.22.2 github.com/longhorn/longhorn-engine/pkg/backingfile github.com/longhorn/longhorn-engine/pkg/controller/client @@ -203,7 +203,7 @@ github.com/longhorn/nsfilelock github.com/longhorn/sparse-tools/sparse github.com/longhorn/sparse-tools/types github.com/longhorn/sparse-tools/util -# github.com/longhorn/types v0.0.0-20240510221052-ab949bbedea3 +# github.com/longhorn/types v0.0.0-20240605091135-ef450e1c04cd ## explicit; go 1.21 github.com/longhorn/types/pkg/generated/enginerpc github.com/longhorn/types/pkg/generated/imrpc